{"meta":{"start_date":"2026-09-22","end_date":"2026-09-28","start_at":"2026-09-22T00:00:00Z","end_at":"2026-09-29T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T08:55:44Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":707,"failed_runs":585,"raw_occurrences":1385,"matched_occurrences":1349,"runs_affected":549},"rows":[{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":159,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":159,"after_last_push_count":147,"also_in":["STG"],"trend_counts":[29,56,34,45,24,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-25T02:07:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-lbsddf with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8c360\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lbsddf: failed waiting for cluster=\"np-version-upgrade-cluster-lbsddf\" in resourcegroup=\"rg-np-version-upgrade-lbsddf-tvrng8dc2496\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\",\n      \"name\": \"dfd3731e-5e9b-458d-928f-6eea5509a636\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:59:11.878359495Z\",\n      \"endTime\": \"2026-09-25T03:28:13.63075435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-lbsddf with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8c360\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lbsddf: failed waiting for cluster=\"np-version-upgrade-cluster-lbsddf\" in resourcegroup=\"rg-np-version-upgrade-lbsddf-tvrng8dc2496\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfd3731e-5e9b-458d-928f-6eea5509a636\",\n      \"name\": \"dfd3731e-5e9b-458d-928f-6eea5509a636\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:59:11.878359495Z\",\n      \"endTime\": \"2026-09-25T03:28:13.63075435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cccea0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-w9hkzhnrs46x\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\",\n      \"name\": \"7e53f084-0836-43b7-813a-0f6f679c2c37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:59:01.927806669Z\",\n      \"endTime\": \"2026-09-25T03:28:03.710517796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cccea0\u003e: \n    failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-w9hkzhnrs46x\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e53f084-0836-43b7-813a-0f6f679c2c37\",\n      \"name\": \"7e53f084-0836-43b7-813a-0f6f679c2c37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:59:01.927806669Z\",\n      \"endTime\": \"2026-09-25T03:28:03.710517796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006df780\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l2fmpdwgxxr7\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n      \"name\": \"c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:58:44.421041016Z\",\n      \"endTime\": \"2026-09-25T03:27:53.514379412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006df780\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l2fmpdwgxxr7\" to finish creating: GET https://rp.j7908608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n      \"name\": \"c2db7371-9a1e-4204-9ef5-6f3d7af49462\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:58:44.421041016Z\",\n      \"endTime\": \"2026-09-25T03:27:53.514379412Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"10468b2864f877d33d6b2b49ecda8314a65d0f555b80b14355c0af87148ea4dc","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"22cf1e89265cf76ccbe0f2e98d44db46e9295497f14a6e8d772e76120a16dece","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"249f1acbefa9c6f20ccd9712fe5a00c23629ae49fcc09ce711b79b24927d99a7","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"2adebc884c23131da1cb9fad9e2d7884273de1cfd5d252f83e841782e4127ee6","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"2b35303720f768483a8d89608537e77ec72ff8a2dbd2f717eb6913e0b2640354","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"3c5cb05e3d81db1356f6fde7ea0b62ca1261edaad85a2845e3986d212eb6d0d0","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"40caa6e5062642065971953209cdd8c9b48b360fb5a1ddc9bc485ca1336da7db","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"4d22a3664875424009f741d660d7e41b3b5f053f4621ecdadc098445ae1ea436","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"4e3b027db47198851dfc1a758d2d2d219a8eaacd78a6c47f12f5ec4d6ce3fe3e","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"5b5cc3f029412de7c3201b6a756aa4d38e26a2737b8850dcb0ac5cda518e740f","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"5fde953f368ed57b002630d1dcf62b22a35d8dc85adbbb537362ccc55a78382e","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"65801013559be7bb9d6cbeb6a5a16a7e16d2f3de96a3532698d9587676ab9892","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"718929c4f69b8ebaee796119ffa00a30eaba1b5c81dc9549343d0c9d2b8dc5f0","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"772419b8b1da0a62ac9ae138152e4f3ffa0ce55f12a177fc428c05293a97832e","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"85907840a45a76d3a4f061bee5f17411e53f0903b97983b05c738378f6bfed7b","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"8dfd2d9aee99bd2f3c4b3476e89dfe81012a96f67d220278846c2a4450bb6f6d","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"a7ae7c35d502ce2a586d4148a43a08195b73b6c1310e43fa75f4c2e07e6936c0","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"b4611c05f2efe3971dd158997c067a5564d4eaa50f357276c0c5460a0c6e6a6c","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"b604b39072c658d2ce9100ad7c38c8840c663636b574cfacbc58b604945878f6","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"c8027d91e14911182feeba1f8b9551aac196e52d333c551356e8711f64c54816","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"ce5516485360ff5329c2949a660bc2dc211057187d80e612e660b2bc332176ec","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"d0c345a150f7ac5287d9623336261f3cc51366673893d4967734a655c83ca35b","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"df135d2d019527778b2ef6a5cf1bdb06fcbe11f52a39c8f44008c1c975f75b8a","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"e6be7f265e4d9ec98cda597c7fc56a6dd0f69b9755ac6a89ac9f3a58964311c5","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"038937131ff2f8c58cc268033cbe88327c0ecf6260a7273ef6763e38ee56eae3","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"0da4df177339bd718ad09c5a87da0bc13c7840ad3a67b75bdb0840e9e8d78aac","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"114c59c2bf6570c0eefc1b7b5e5a65d9ac897e0594f2a59218c2788db88a3400","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1472d735a7d85501b8306e000dd5d8b6374ba7f340c2e4ceebc2265fef3c431f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1c0d63e4358fa1d03a508395235eb4276e75a2a3b1c57828b7c15799aa97cac3","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1dbed21c30a383bd27b5dec9ae60753fd2bb2ee86f63bc372cce6f9506e12b4d","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1eff726a97351513b4ec821562ed516277791ca031fbd1e1aa1de9c857212fba","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"35ebec740614a91af05434491fc4858f3e451d08e674d21eef134a37cf5d6508","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"3ae4d5da8c6399c11ae57e63b6c3ccd31dbb7427337ebb04cd7c8172656b71c1","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"429c76b6d09d4b68e8144f42b4ace52233d91d9e8ee5711e33046c0d5d3cacb0","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"4725982cb9ff6e3263bd7373f034c89fe01c51d46047009fa1ed4ed4ee62e38c","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"500f59df8165aef3ffa121de8a9c035e4de2dc3301c2c3ddf3ea43873d0b709c","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"502b4a1a6e0a45081dc28c470a10f7dc5d98fa3c12fe64214fb3ad5d8b1d6533","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"55c4b969417e538731e376035caad7977a5aca6f1f2c72913b4936de93c287e7","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"61d2a995fa7cf5627014299f798acd76615bfafedc8979ad3e36a0eb0c7a2662","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"61d69de89a82efb1426dba6f56799b44a42e7adce363871a9a6a5cea8d8616fc","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6502c23f1e7ec0cf695a6476bed4b50f740a286a0b7fe2eed834a4003a061b59","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6e4748bfac20ea730efd2ae4f326feca89e989b0d1171f412e6d519f9e1670e7","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7418caf66506fff095427d27ff9a805df356854e0d321f2288eaa6d95d485b6b","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7517cba222e621154182cc4ff18e45927f0a4901a789bfab867f7abd52b27ff1","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"79eab85227deaa5a504f26b49c3794ac1f2793daeeccfb151c2ebda50a94f291","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7c3d27901cce4d1113cefa72f1c0a7197bf58cec686d39a74e26062de2c2d826","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"878621b3a630c1b8725c92562eb8fd6ff7e73ca262fd7d2d1a4d3c703bd7181b","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8a6e31d88bf2e3e5d322a70ec29117555c04f8de5f296cdc0c68246bbf060ddf","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8f058c62c69f94c523da555f854a97a9bc6956f04a0436c7a2ee43ab827dddb2","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8fc3ce0aaf075ecc748035615ba580d85a206c822b6daabc625d94f5b3a03ac5","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"91bb81a2eca403f9cb3887462e7e35230600fa3a01fbacce487d699ab1f7ea14","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"91d967ad1e2051c89285c002d456dc98a5e8ae32206d5cccbef47bc1c326378e","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"943aa3a834a0c2cefb204a1c45c66cc889d4122f1d10900e7220b69be2bceb26","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"9efffb68dfea6877253ddef458a441c37343a3bffa28ae1e9b9b91339b87e767","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"9fd5e8c5641bc6751859b9a36c13659f53bb439f0e7a58e1cb5d575101bf0bb9","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"b5ccca774add3458c9bfae3e4c9d277eb98b1b593daf2a2bb7c71a1a8090c02f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"bb3afbdbc42ca49133c243c7445c3bbf9eb564d2c828e805ba0ded344b330675","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c445de35915f6d6808ebd93b49d8b192fef5dd052bad1e1a4994444e170535d0","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c44a3318beb91865b30052009d15e7f5adae10dc8a6c1f345d1fd64c63b78b16","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c5761bbbe7076b8cad854fc72af7546a6a999f18ed1f68b4bec3edea52bf3259","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c92e73f72644876ddde62cf422ddb3b39fc8bfa8297fb43d32cbf01215c8b722","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"d92c446c7baf0ccb5ed48e5e01951aa4edeca26f9fddab2165a4455ffedb13f5","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e1aa6d1a4b4a58d1ebe60d873db4901be2203876d71301a5e963bfa6dd923550","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e4af5e82f7cb726f365dd5dbed7b27d574f194386873276eef8bf8579929b089","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e538f2a0230f20027d8293b5a6c68382ea045c9e1651af9ed3dd5ac5b523af39","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"f1c7a583a370723cfe1f78b08a12ea6d5d2e3ceec2dcab1ff58a6c2cc20ec68f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"f7d663f4b5351be1b0ab0e046173f64d43548235e11b8e908df07ebaa35514ad","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"fd7b575d29435c0f905dacf7124fa2c4be1abd2c35d8c3bd9219b47920c3bc47","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"ff19a7ca5677fbfdea92fb5506f95dcdd516adf5a368b611a7ee13809b4061f3","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"0a6194beb78b8c406f21d0c39e8f4938c36409cacbef3176434d6c29c859c82b","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"0c65583f9cf2766f075edef8345e43c644c0d3a9639fd26af02b95e481bc76ec","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"4342f7d49a31fc62a7cd4188188d768f7c7622da6926b4fcd1d52c2f7741506a","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"7fc420454baa557ad64e773af2ec483367dc179f35828ca014109dbc19bd2053","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"813c3b4a34551005c00d10be781d74a14c7bbb31e37f764d60afab74ae06652e","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"9eb53b927bd56443c24b7b20ca5bab6adea62beccaa9c3e083e3ba49ff15f1b7","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"a08dcc436350e9ea389eb3d5efb0662df729d203d30b15f5920616ae5fc224ca","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"c525979fb2d08507f3d446b04e74f3e5b602e0906716c705201cd81002770ba7","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"cb11f1de63b86677e85bb10ffef4875807561668b3123fc194016353354c7ac8","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"d8dcd8028ce1adacc03f46fb0158f289f63ae31f72e8e4ce40b790b255cba87a","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"ef425b9fc641abb8b932709dc29ce158807de6ae50f47a33b3db89e10d6a6b93","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"0adb4bb4ae9202e459b2315ccfce11f495585f3d7bac3fc7a2487d9796add3c7","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"1b19c6dab2da849dbdc19f25d8a7c8a64dadf6697567bfe5d10096294dd0a6e4","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"2df52c1cafa5532c454b36cb7b084b68593cebfadd2864d45dd1a118200b8d52","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"2f81637f8265c258ea75aa24f9fd9432ef36114b9ebee46504be7d5770c435b7","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"36a05e474622bc3fcc801d228958fffc4a6d075602e55266d09e5b2e91c19368","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"381000ecc9d6086c37b0219505b21d93a9c939a2ccf23f03576b38ab84b5a55e","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"449030cbb33da5d4c8dac990dceda44dd62ccc5ca1ffc1ccd12391d7b55e0681","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"47903548f1f143d7ab9998fc3c8c1fbc469323e86e2d9a0d0fddfcc0a731af5d","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"51f8c6265949434e6442ba80f7ade4730dcad12175f31515ce73c25444d377e5","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"5f3ee90b05be0d7f1487912ea92f8d3f0441c3abfa713a796fa8f25021c270ac","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"6fc513af79b2809dcb5d5df1bdafeac7cf5d4c3a1862412ed3eba47ccae158c5","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"89edcec5785330a2ee63b672917ff746b2f687ab7ded705d0cc185c95d8bec14","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"9731f91656941bbb9380fa37cca11d45aa6ad2c56b1236e28cb4401092e4c0e4","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"97bb982d1e6ea1aa19045a58c60ec413fe3ba3d8834e3141a02e5782483f6785","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"9e823b348b35d763f4f7278b55c97380508738106688d0cd3c61c152c24157ec","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"a465030c91e7749c08c2c7314b00f14fac042b380238eb18631f6fc0cc7ad978","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"b4f75fac021e402acc2db264f349c96a35364d535bee0b2e0065ea599920a92f","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"cb7663c2933e4c98c3163332183af2e5279eff6073b28c84e67ab60d8693fc1a","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"cfc810c40e60a646985d0ea21bde9b0fc719a1179e342cbce3fe6eb38a62ed69","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"d6cea3ffddfab34ef89656bb4911dcfd537e070bc053f39a0b804e5db90f21f8","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"dd70726fbf025a2e22838d8e9a9bcfaa30032d7ec07810dbb606326dc89fe874","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"e1d6b4b1a86d1907639ae72cc2ccc8137426d24a60ad5b4f6f3e7247ee41a9cc","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"fb3c31a801804fd94e8c51eb7bcff2f2193b26a3f6b7ab9373ac03f4a1eac932","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"096fb94a6c94af770de1adc257d2294c1a5548271d7b9aad906d0c0a59e70fff","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"0af445052595e06d74cd6d8fdc079d5bff2df4e78361aa64d110c0c33f929996","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"0b2a132ab1552649222c76502236db8e74948cddd63f8f30c41982bda0fd9971","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"1526d73081656cbe45f7dcb1e8eda1497f2bff26b20c12f7d59db1211cf00231","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"1c3157c947f58fd7bff96c9f69e32f40095840b97dafe53456bc1253bd4040ee","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"1e189a19fae505de745c69dbb168a51fe98419c9e11747f4f79c161c6db4e7ca","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"27b5b5c1e8a00a07e0d9a24204eaa220c56b06307d69b9cae9ceede561c51215","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"28a878be48996c1a5644504392baa72a6c06a93c17f7a9adb5a48a03c1f748a4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"2e05c40aeb58cbc76cefbe3d60e5dd6051b60dca39a8e9fafcc5368faeafc26d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"3702443bad4883b0bd4d8e1d16f90d0834ee6fdc57f03270085d8eaadca8af58","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"42781e15bc1b77679bbd1fda7afcab888bd256804d8bc00b2e3e391e7f5d9698","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"4f2dccbb23c4d3203abba3ef48c29b8b62112b27094207d9ce163be8ca02b232","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"5202c14b8a67b24502a01693eb0e6502a9081428fcd7886134dd25c4ebf94bdc","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"58aac2aff6f1ae29ed7c65f0b97e2df1f1fdd10c28f77f899c5d1127d0967e40","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"58f2b440a994a0c878d24293dff5f6b4514980f374d184722cf27fc196af816f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"595136f34e3717abb66fa01ecb7cbd2f7ea1e74ae44dce8c1a0aa154c62f402e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"595498bacd4e938bd044f59e576c914a426f362c531efc177db2352d292b3855","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"5fa71470de1f76cf4fb095baee252cc2163afa119633dbab401e106d7fb2bf18","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"623a2d85a1841b8bf34570ecee70c8bdbeff7f363469e44c0648673b157343af","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"667049967e319dc03b972259bd67db46a9a1814fcd21bfddde8fd56aa73f0ea4","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"6816e0ef7bc04a15cb8eef8e32843b61c4fcdf6e64b22c55680deaf9f1aee076","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"6d0286b4a77d24ef7787809bf863c3ac7059e4e55a1c01eb88019f4277fde825","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"6edd720649594e35198f47a4712e7d929e2cc171394ccec97afd8ce85932120e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"6fb962141bed0f910461aa1150d3a12d1c1dcf673c5c5a69cbd92ff5968cdb05","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"77d324f886ea8df5e4ca54d8f6e4c7809e792140fd13f531964f9a2e4ad353ae","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"7aa73d22a62996df337f4919aa5b6a2b148aca5d4d6e3c14a3ddcf0f290454af","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"804139726f070b1041f8100fde4c47aa1c25e4dfc0c87f55286bfe671852281e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"8319add2c226ddff82deec727b005791655329b782a15be502de9fbf36fde07b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"879e3fe90aa85b2e72d47865b800cf4566cf8b2cd76a787b032c5fd1932bee1d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"8c36d32a0259f7da9b9f1ef40b52c366005c76cbf09bab0837045aa60989b093","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"8ed34e522350ffda7fb250647bec9f130736688154ac90808bd25676fa118f78","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"9232a7400391f356316f15eeb51fcd91beb1f25acba3ab886fce2c7e5b56d1c1","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"a08aba7b10f89db3de8a4b9e00fac22f367da0ace6bdc0a9edfe1b4d61fc1984","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"a19b9c1a10e185f952d6f4d9f82ce909f40d51f7f31c07bb14777fb31726a16f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"a1f111f15dc32b37e44ac7b8ec2b54927129d700f72b123fbd0163455608e4ab","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"aaafc5d30addb61ff662830f1217bc76ce43a223d5f303ad8447f9d259c6bef9","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"aab74da0f5bf35c1456f1863c7d9823e709750f96f75dca49cc4b64a53f482fa","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"ab7ef6dcf373c206c62413a791dfd97f4b51a92220f453b5335be9a51f844c5f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b149cb73141f2a05314c417a812e2beec7368d22e86ab78a853852e548e0e065","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b2ca19944d899fbd5a1416f0e4d1be6345d0d373d8de61d38fe13164fd82111f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b5b9ace7c0308fb8f16c98cc948db5775855e4b1dcffad726de65815e01eaa00","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b8806b22f97f5a8d736ae5b95ccf82001ed6f6ba3eeaac0cbfbc48297d8b069d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b99e1feb6a62e4be8de2a2c9f91e219add3d2ebb4d288578917ab29ec2dbf2e0","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"c26d00b8f632469299803121e5e542c1bd6d48e71cf443b1fc9c6b56a9980e1c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"c8c1a15166f9304465eaca377226a0779617f87f57e8d7a65ecbe7dd9d745a90","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"c9e77ba097967d05f7bfb10ea65f895976153a5d4d5aca8181798ec5188c475d","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"cc144734f26b1d2b89daf0a0292b0f322bcea668bea3417d61ea7b6957d358f0","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"db89aa3bdd096aab58b24117e783390f8fd4094ede4f720ea86d619334ecbdf5","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"dd91c26e3cd9c656ccb1a1a45547d7332939e80b90ce720939d4434685892e6f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"e56e9654c5bbe6050e3fd8a85052c273961dcc81b47c66b91c206ee1411a60b3","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"eace06cfe41d2ebbae3c6b22326ca19f7591d166aa15abe4fec7bfa0baabb41b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"f6d6a168159422ba9c70117e70748b52c1c05f58d631c53185b407f05a9704ce","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"f790b8b598528f695078414dcffe0f7fff8c6bd324401299d0482a3254ab3f04","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"fb1ed664d9909c01299048a39e7b19384c539f31e18fd1ee83b1cea24e43310a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"fb9c548889e979624bf5ffaf1b274586a10ce39b24af64381e4f94d6371d35e7","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"fbff5fde4987fdb4c2873cc854520923ed13f84351aeb3493ea93be60c17edb5","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":154,"runs_affected":154,"failed_runs":154,"run_impact_percent":21.782178217821784,"anchor_occurrences":154,"after_last_push_count":85,"trend_counts":[20,21,22,24,30,5,6,46],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":211,"prior_last_seen_at":"2026-09-27T16:28:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":154}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T20:22:39Z\n  Ended:       2026-09-28T20:28:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9312256-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9312256-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T19:18:56Z\n  Ended:       2026-09-28T19:51:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0189952-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0189952-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T18:59:41Z\n  Ended:       2026-09-28T19:05:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3028352-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3028352-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104648687849312256","occurred_at":"2026-09-28T19:05:24Z","signature_id":"5ec40f1dbe64ce284597d9fca92af72b5138aaab20deba744a9e8c7388175941","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"3bdd3d319d232da25d5a6952031f06bb164d2fbc1f8e8e5f1df140b9e37b67e4","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104626712733028352","occurred_at":"2026-09-28T17:38:04Z","signature_id":"ad745131b08863f15bc2d4da07c88878d389ce2e6a6b3cb61447530799f60d59","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104623441356263424","occurred_at":"2026-09-28T17:25:03Z","signature_id":"874dec276004f85c33608d553f88244c23c135754a3f5df97b8525a573d31443","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104620444073070592","occurred_at":"2026-09-28T17:13:10Z","signature_id":"ac6a9a0ba19953408fe70e4eaa33691a22600749799a03e7a8a0d5f34b42d29b","pr_number":7227,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605657771544576","occurred_at":"2026-09-28T16:14:25Z","signature_id":"fd2822c3f7f785cbd223c73c54fcf972a98b49ae3cb4a8e06c0beb83985fde95","pr_number":7104,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605132682432512","occurred_at":"2026-09-28T16:12:19Z","signature_id":"b68e00c86c872a4db070db835721b53859e77302e16ae329d88531395a8e4cfb","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104593036766875648","occurred_at":"2026-09-28T15:24:16Z","signature_id":"1fcca37ac24d8001a48be0c5ac838fc8ff91d02e6a09bfd794ed9b778404688a","pr_number":7224,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592047892598784","occurred_at":"2026-09-28T15:20:20Z","signature_id":"01f3b1b330714082cb25db8e5c89395b0eafdbb7c5f976208821bbe7acefacea","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592038186979328","occurred_at":"2026-09-28T15:20:17Z","signature_id":"73fe2c4208f24efcea5616c31f3812a95478add8edc71b52d609662bd139ff23","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104591950119178240","occurred_at":"2026-09-28T15:19:56Z","signature_id":"0fcfe2b08422f9a39332ea81e524fadc56ccfa65ec146684784a80ed12090466","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104588552477085696","occurred_at":"2026-09-28T15:06:26Z","signature_id":"8bd6e6bf74442257ea4a8d5df10532bf229f7a19b2e38c9f2d2d23e12d0aa5a0","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104587545261117440","occurred_at":"2026-09-28T15:02:25Z","signature_id":"bb22d546fa0f1214705bb254ddf09be8608855766ac347879263e53fc41c3caf","pr_number":7214,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","signature_id":"856bdf33775551f2ab32007205cf5d6ea2f3c3f28ec94c1bfd6fbd6d4df71cdc","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576830420815872","occurred_at":"2026-09-28T14:19:52Z","signature_id":"b45324aaec7eb7902136101858f41a6bec16eed34781ffde48402ce2ed72f330","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576709901684736","occurred_at":"2026-09-28T14:19:23Z","signature_id":"637d9f518ac8eebc8d39c5a4b7ac660a4f39e458e3c7f2cb4b55812a9f97d2ed","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104564648333283328","occurred_at":"2026-09-28T13:31:27Z","signature_id":"6588d10f1b9c7b543415a80c3d0f51e6af81c61ab7d144c6b3dce965ff9765d1","pr_number":7225,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104561257146224640","occurred_at":"2026-09-28T13:17:41Z","signature_id":"2849f38d4e46e23a374c8bcb9103369c6a91ad3b81860cb4691446e160ba14ef","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551522074365952","occurred_at":"2026-09-28T12:39:18Z","signature_id":"c8b7bde7071fc9b70fbf2e0c3fd7b6a18f6e7bd846c88f638e2208d436b6bc37","pr_number":7221,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","signature_id":"0f15772b2206346fb5d44faaaa43fc1a23bbe1d33da8dea71728baa03c962358","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"44087d2f83ae7baacb227e44ff9437079810ed6d36fa5babed386db17625fc3d","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529106451304448","occurred_at":"2026-09-28T11:10:13Z","signature_id":"a482699276ea2332ae41de1259b09677409d1eb32ebc3d34db43a27f7addaa39","pr_number":7182,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526339443789824","occurred_at":"2026-09-28T10:59:14Z","signature_id":"cd68e3e4b37b20fc25bbf6e62af045b96ffb4fea1b64e10b73cc04c10aa0cbf8","pr_number":7217,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","signature_id":"c224092c3a3b7cc45c3855e0f2ef222d7078ecb744c1f53a5b2ec26cbbaf5fc3","pr_number":7218,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500979540955136","occurred_at":"2026-09-28T09:18:22Z","signature_id":"8ee7b403e6fe4426552b84f141c27a733cfae0d13bff73799620b851e1e404d5","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500972830068736","occurred_at":"2026-09-28T09:18:16Z","signature_id":"0c3acf11461e0fac9b5520d458eb6f01cdf27f718bc25ff896d1d3bf8a9936d8","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500902567088128","occurred_at":"2026-09-28T09:18:09Z","signature_id":"5c7d7ddfa07a3a6a4e8abf4f2a8fa726968d0d86dcd83659570e6e8d5c578e36","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","signature_id":"8ed8ccf041f97eca095f371db1f62c00c01ed1a5f3e20b41eb9a02f2e97c9851","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"c5c86d4d0c9a75c7f010a1f205acf66c6c18105fc74571e1cc5f8325e06c6ded","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484432940371968","occurred_at":"2026-09-28T08:12:42Z","signature_id":"1200d6401a7d362c5bf39c4833d4b21acde0146bd26ab3743b40cda347a928f3","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104483794588274688","occurred_at":"2026-09-28T08:10:10Z","signature_id":"6b427d606ce250ce673bfc708c7544504d7aa9cf46f968e353a45fe179cdbe65","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7216/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104472130388234240","occurred_at":"2026-09-28T07:23:49Z","signature_id":"c01af702f03601a1f0d4848381ba9bdaa5bf0ccf7799bd12a17435bb2cf8037c","pr_number":7216,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","signature_id":"ec67058d65d185cbb1282193881a768a798758cdd0d0444d20953e4ea53d053f","pr_number":7214,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"8e9a5272f264a6a6fbd212995fbea44c0c88cab133bdbe6fddc3a31df7edd368","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","signature_id":"0a61a6329e586b5af75e7ba37f30cc0d75552d5e500dee8165072e8b2848fbf4","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441035131195392","occurred_at":"2026-09-28T05:20:15Z","signature_id":"679e5de886d8c0615ef4790483cd7a9e61e288f3405b68c87e65af78de307ef6","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","signature_id":"10d54fdb3fd853f5743498ec14b91987dd77b2bc92079478e8da6f76b7e1d6bd","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","signature_id":"5eb0f55666a6428c416d4036a93a530ef71389fa1bd3105e20ed8c531ec7abad","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104433568867422208","occurred_at":"2026-09-28T04:50:35Z","signature_id":"311fd8960a4e8d6a6edebec933ada94dd10107fdfe37f5ec1b9508c64913d1bd","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","signature_id":"e707533b83464f0017c7724f1f2fa89bd9f8f29f5f20020a54be5188c8c5bf45","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"0fe28f25b5227c777c5fca5ce997d516e287587c33aeacd91934a6120a38de47","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","signature_id":"c3b2564dee31ddd5931c6c40cdfa6bad3a8068969e02001e0d6ca7548f201b91","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","signature_id":"303fa1869053f8ae8a6eb8a08cc98dd837abeafeb2f0bb2dc5dfcfbf4756a32a","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364960745263104","occurred_at":"2026-09-28T00:17:58Z","signature_id":"08b361b6ce0293b567f62579f1da3a35e3254e18d2a7eb29bb5f9ec42f4e8883","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","signature_id":"846cd81d2bf49e1ba6754b4c62a52bdae73d630e941ce6555ae43446efc0f737","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","signature_id":"73e11070f2a858d26270686b98d03cc56fbf8cc31e4b226ab179fe754a016a71","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","signature_id":"d18c749f3722a30d0857576cb4260959235b0821801cb56556539b905f76002f","pr_number":7199,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"f9c445cff424013d845cfdec607b448ab1198b54444d73458cb4dab5656a8299","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"430c9f14f7bfbeb1599ee7c1c8f52ce30424b0cc5c567fd877a045c6cf4ab795","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","signature_id":"c24afa2c3a89b1db96f561ea699ffc16b6ecf7c1b6c7babc01f986b2e2f32c4a","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"1a89df4c7c180c8de0615a9daa2555665cfe23d16df3d8e06611837f74f549ed","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070538413477888","occurred_at":"2026-09-27T04:48:02Z","signature_id":"84013e20db593dd86936f009da2bff278404d255c4473789a9a242cf0c678207","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"ab36f01ba1f6dd47c0278082b2e3a50d4690b691b0c683c8e73be1e12cca3d69","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"342e9a20297ec559151d137a61844f3c9a0a2d3d9dd6a479df3a39341749c146","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"ebafac72ebe108727ebff5313e306f4f9049eb95d8f8e7da4ad87e9d88b60349","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8e031d136666491c65ac1f03120c0173819d20c1c3adaf01d791c67e1aef0c7c","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"c4223b7b494a6dc2601c8c97e4b1d46697163b469eaf658557332ddc1cec8106","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","signature_id":"86cb6bb93c82d8325bd82f0286724ad121ef3f1b1d15c8476cc5376871629ce1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547560068124672","occurred_at":"2026-09-25T18:09:53Z","signature_id":"07b639689eadc8a3b695373d070cc3ec10682dbbfd11436fc6587f411d82545e","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","signature_id":"6072cc6e524eb8561dccfb6e08abf762fc8d80b75bf6be872960a1df3249592d","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","signature_id":"bd72e06f6355865d4197eaf172fe11459b96dc4ec574f9d4b405ece644f53733","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519728415805440","occurred_at":"2026-09-25T16:19:18Z","signature_id":"e3c1944ee456bdc02cd730f513d15fdcf94aa9db83c39ed1a308fd06ddef0926","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","signature_id":"4a7342040e61029b69f8b6b07ebb37773b7c8fd7a7d80fc00ca7010aa64d8ffd","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","signature_id":"b81daef2683367cbd95fe9df79dfbd725276998f256d289948e3586da1b5177e","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103518634033811456","occurred_at":"2026-09-25T16:14:55Z","signature_id":"2a236b2c0e6b67c057608ab27afe1e5a63ddf0d896563f1d1f4e9ce94cf1fbb0","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103492665126424576","occurred_at":"2026-09-25T14:31:46Z","signature_id":"9fa96b5ff725dfc0ca78e5fc48314cb4f2f8426a2fe124c1cb522fbfaa4d32f6","pr_number":7146,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"c3527bcea856a29f65c003d1a332595bb1db06adcefad7894814a3ada10903a6","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","signature_id":"6885dc46c8451b99c4e7c013957eaabbce1c6b219c76ec914f0abe3783c0fde2","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103462425171857408","occurred_at":"2026-09-25T12:31:37Z","signature_id":"028386accf8ed10d4b0a82a3fc25eda87277e1cd283cfc4e15e27e0fae5cc1db","pr_number":7155,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461761565855744","occurred_at":"2026-09-25T12:28:58Z","signature_id":"9dfe5be11daafcf9e81f7d064effd8fc3de0a5d41bf293ebd6005bf43e5445b0","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"ec90986a9952136621543366f85ab08fb1261aa887e4625683d776a59dc714cd","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","signature_id":"8e84a5dd9589dab0c443e45f6f89b76cc4dfe1440ffcabce2df34ccdfef20636","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103450358956167168","occurred_at":"2026-09-25T11:43:23Z","signature_id":"f744a6565b76474ced1ce309d8efc72176260c6843e213d9c3fca66de936d085","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","signature_id":"c4612c6f0c4f97aa285084bb11c15657a376a3c30bfeeb5dba0a5261ddcef8d0","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103424207261011968","occurred_at":"2026-09-25T09:59:45Z","signature_id":"d9de05c180e826fdf349a0c3e79eeeb21bd08be7c8f2912f4e152b0c6dfd4d1b","pr_number":7003,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423928134275072","occurred_at":"2026-09-25T09:58:38Z","signature_id":"b16f25149b765227befbdd2221796f62e8724e78f44f61d5081a1d60dcec87db","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423855413432320","occurred_at":"2026-09-25T09:58:21Z","signature_id":"9c9e2ad911e851ab394d8666370f122b71b040daae5e9bd186a4b2a374c792dc","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103413943887204352","occurred_at":"2026-09-25T09:18:58Z","signature_id":"726c0aa747c8650a3d621f95eb1b0811a06db840cb471b91ecb3ebd116e75b46","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103412592713142272","occurred_at":"2026-09-25T09:13:36Z","signature_id":"8e8218e7a5915087525d6f8f3a88d4f7f906cb2241d2287308990610b3c657ab","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411013251502080","occurred_at":"2026-09-25T09:07:19Z","signature_id":"1951379324799428f6c90ebb62ba30f9313ee199425535b87b2aa1e4bd2ab1eb","pr_number":7071,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103408234294415360","occurred_at":"2026-09-25T08:56:17Z","signature_id":"15a9e409aaf5d9e0579bf269fd83ede9ed5d3edd62cf633152f5854e716b53b1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103397005047894016","occurred_at":"2026-09-25T08:11:39Z","signature_id":"b6e3b81994212579ee43950ada16381b253fd6d13b3af77560b9b2fe98d3254e","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318253316083712","occurred_at":"2026-09-25T02:58:43Z","signature_id":"0b5e495e1b35ed4266a96aacd072322525c0cf00dc1a7b76b0d2653c3c70a28a","pr_number":7150,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318216586563584","occurred_at":"2026-09-25T02:58:35Z","signature_id":"c148dd930f4f1038c64e5bda079b2babb2225218b71356ce91b62e5328d1c0a7","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318217182154752","occurred_at":"2026-09-25T02:58:35Z","signature_id":"f4e4b5d37bf82f3189cfb3aa38f15d4ff310e4e09323e5b109f9510546431a86","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318164392644608","occurred_at":"2026-09-25T02:58:22Z","signature_id":"4935221bcf4cd8ebd4a711d2d33e112ac42f8c8ae29a231910ec6bb2fc856310","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103304771547959296","occurred_at":"2026-09-25T02:05:09Z","signature_id":"4f8032ceda13192ec98e764421a26949f70feecc56ae688c668ca100e00819d7","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269477926834176","occurred_at":"2026-09-24T23:44:55Z","signature_id":"6f2b6a144e3541517f2630051f3cefcfffdb5893b1112dd3f933dca064c7ef54","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7145/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255589084729344","occurred_at":"2026-09-24T22:49:43Z","signature_id":"84c6dd2f539442a735ddad2dbf44c1744086e075f67b8f3edb677f60e9a862b7","pr_number":7145,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"d8855120e86c8409ebf1202c522f0520dd727db9ff94a1dfff81d2c5c15fcc43","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"d06adfd8f5d6c7ecc9b42d9b62223e573b7f16b0df6ea81be882241614b5334c","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","signature_id":"cd75991f1a2d3f936234a24c188b3b95270f16a178b722a2c9be2086d6903383","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218829420138496","occurred_at":"2026-09-24T20:23:36Z","signature_id":"09e1889769e13cbac0f19e0bdd92a9ef75986032637b4b9ddaaed3dbdd08afff","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"8cd39d3a27a9983e4c03864ddd77a3a279fd3ba43a01731a97e5915930fd1c60","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","signature_id":"3ecef63f876f1b80222fe16ba8dddbc997352451ab492a895459bf901b651a77","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103170613366493184","occurred_at":"2026-09-24T17:12:03Z","signature_id":"f850918927d721d1a9c993363a6f66eb1a0457832ced1c9c9d5463e320ffdb40","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161541376348160","occurred_at":"2026-09-24T16:36:00Z","signature_id":"2c5be9a399728a0c355066d00beeef8ebab315261af0ce0681e717060c90e781","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103155003655983104","occurred_at":"2026-09-24T16:09:57Z","signature_id":"fe0efdc1006e609d9473efff132bf7955c32a6ea999c6a2e2fb575020cee0852","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","signature_id":"1d82549097eef2d75be7580f57f3df55a47f5833cfb899cc75ff5cc5acc415fc","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"342114364ff6caa404513f6899c73460c984413773999113333d0dc8ff64a1da","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"95d616d62fd8b68affd2cc0528b1726d4f7bac8f448ad84828096223e0ed6a1a","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","signature_id":"0316ed16bae2b06726cffc3f37180891ba349c08b7cf18aa3d8d47e6021122fd","pr_number":7131,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103109521260941312","occurred_at":"2026-09-24T13:09:15Z","signature_id":"d8ab59ad81a765023a17dd370421f6eb9d9e6dcc676911949deb1b2a6ad616d7","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"6dc0e8968ec60f0324b6d07caa91d4f95c59a07d57bde431637fa99a19424678","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055091593383936","occurred_at":"2026-09-24T09:33:01Z","signature_id":"f048e9c9961e95f1fc58ff19153a9940b10cb95928a512cbddafab4912d29e0d","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"a0b9e359a9ec67136305ac5041742a0b1b9f95678b9b1c144a344ebc7e1e7a01","pr_number":7126,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"bd015bfe1b7ee0302c28b53124982a3fc176057803cf96dcb50518f45695f0ed","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102931134991044608","occurred_at":"2026-09-24T01:20:27Z","signature_id":"5d2d7a79a877fd36a7ce50a19b88b5be576cc7d6481c13905922189eaaa22c37","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"c9c5fd9f1bd379784e5f4e5bbdd029af5ef5b392fcb1f2d33857ec4a703e485d","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"30e0e523f1fde040099a61098a3eec7b89130d7155eceabe4360309fa9b4a812","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"c5508e299b7fba05140b4f25dbef14c99e81680c4220a923738b6a2265553a53","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","signature_id":"23e9b2995fc9ded2e9caf55c0eceabc998d92c6176e04363cf44c1eee0ec317d","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"9768ce66171b43df472ff44180bc3d8e585fee001b4346d3ab9374574a7b151f","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","signature_id":"766b1d46f5072f7c906c6126dd0836e5f198da05367c355a94ce2d777ad7547e","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"56b2ff341978727c646e219e23fba2dfc1ef2e5ccc4828702e0dece8d2810ba6","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","occurred_at":"2026-09-23T16:39:42Z","signature_id":"aca35d9e741dfeba834ff114bffafa2a455703b5a32f415b37bdcbd261eb6a67","pr_number":7090,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","occurred_at":"2026-09-23T14:36:53Z","signature_id":"3135ce76bbdd1ef05af982db7996ed6329db94eadabfb94067b89a304dc4d2cb","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","occurred_at":"2026-09-23T13:56:43Z","signature_id":"b522a7cea6c2812e29c1ea3b0132f66a43bf87236c4f0123b2bd527a73ab14bb","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","occurred_at":"2026-09-23T12:59:37Z","signature_id":"f10cdf2980f54cb7434c563cea5c00835d86c81de376364c86add2439c33d20f","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"e8f9ea179e852213d3a81d66ef232d1e14041b2de7883a9afe6b09a4073f601b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"66ef3a438e94ca88c6470f62824d14213734daf6ab2142316c1d734ddeaa9c0e","pr_number":7103,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","occurred_at":"2026-09-23T11:19:17Z","signature_id":"04cf9b464f041c4aa5f1606316891b1c2238b1191be6f182872d69d9ca55ffea","pr_number":7072,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","occurred_at":"2026-09-23T11:07:04Z","signature_id":"ecc22c0637cf2c040e23438995345ce727a3526729f1b45629b38665f2e2a492","pr_number":6439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","signature_id":"695f2b58900eeb6ddc3f5c28514b819f16afd04fc4ec5e5122696e6f1efc6a9c","pr_number":7065,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","occurred_at":"2026-09-23T09:32:18Z","signature_id":"5be4b79fd707f0f902f0c6190406a62281b77b6ad2601f0cc1b46e4e850c930c","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","occurred_at":"2026-09-23T09:30:30Z","signature_id":"a430843224ed1479b519c2f15a1c5fc4265a858e6f819daf935153b2c4d45bf9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","occurred_at":"2026-09-23T09:17:52Z","signature_id":"168b46660d0ed6f4bf2e6d271e550940fff7e50d9ae0544d3d77fc3aaa3c674c","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","occurred_at":"2026-09-23T08:02:37Z","signature_id":"ef8ed7f81b6e444cb9f47b23d2f051a1e779787724507e05e79e426102d41cd3","pr_number":7072,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"24389b82f6419bf8544e5612ab91556f9033a43c11c58c8a75f9eddd0d3cb5d2","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","signature_id":"7c059f6930e4e70e01cf60cdc5a209a41e1d36e3c5cdc0a3889876307b50e19b","pr_number":7099,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","signature_id":"9b7c4ba8bc11e40e7d6443d7bd87ac580a2605e35dba3d0581d0e43dcef62d3d","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","occurred_at":"2026-09-23T05:01:42Z","signature_id":"e61751d58352ab491f0598c208a923b5d0b7b5b03f412d966b5363851718b33b","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"b52a16123591cbdade9825eddb9e37de34eaab046973c4b8e648b0ae88335846","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","signature_id":"d5512e2af1827ff97e8ddfa3ae72ff959c7739aa5b0e2ca11d43d5d5a2447e6f","pr_number":7084,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494708675645440","occurred_at":"2026-09-22T20:17:39Z","signature_id":"c65229f531bd77886e311f6f9ef82fd40ff68bc0210a843c2ea5376a427e25e6","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","signature_id":"21b04b293f9281378dcbec21bbaa132777f4a608518f67fcb04c950d001b5d21","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","signature_id":"c84a9da375a2e42b5ebe685d998166743cab67da7aaa85511f0c2a9396b4c3d3","pr_number":4878,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","signature_id":"543e7b8c3c4a28ef5a756b6c10eb0692ee1db036dee1df85b90761212f5b3fe4","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","signature_id":"56de1d48b5dba8bd66ed5eba60745f0f8b40aa65ee4126e312322ff3f116fff4","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409636605857792","occurred_at":"2026-09-22T14:48:11Z","signature_id":"dfd6f97a6359d56dafe4e7fd9a76d40da48ac489a5ba8285f1381f935bfad629","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102407137983467520","occurred_at":"2026-09-22T14:38:17Z","signature_id":"34f1f17119f4d188bf1ad27cd49fa9062927b1fcbdc68b9c068ecb52cad6117a","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102401634003849216","occurred_at":"2026-09-22T14:16:24Z","signature_id":"21991f4941f1b527a268eb9db9017a064881bad0def97631087a0d0253d8adb1","pr_number":7071,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","signature_id":"69a4d01934f71c8d4cc5f95af06ab52f13a41aae602819428092cff5854cdee4","pr_number":7001,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382852271247360","occurred_at":"2026-09-22T13:01:46Z","signature_id":"5a293d3681a6cb16a3acf3c0acdf1d5978054ad425ca5fb3f977d8e2311584eb","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","signature_id":"6c8c17148df8d724ea7e8dac7eeed751ab50ce8837feab77bd06d0501cc3031f","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","signature_id":"625c86184442105b29032ee621018bbc8b2cc587e8ad3237c8894fae23fd25be","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"097bb7e262a51161f0f6deb237e175dc5d5fdb158c34757a9099308fac85df34","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102324260725854208","occurred_at":"2026-09-22T09:08:57Z","signature_id":"214043246b33b8bd15f594a10236827f03be2665055b40a9736925f22821d35f","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","signature_id":"54194dd46ed1dcbebcce9a1d3bc483a9cee37d6fb298a0432bda317c79de57b3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","signature_id":"4b78bc370e58022f6fad2989fb6bd5f3829134afce729259ce754ba08b7ed2fc","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102307266211352576","occurred_at":"2026-09-22T08:00:53Z","signature_id":"006b30f9681fe7ccddd410e35778871e243e77c2a5c3d3f8a93cc41e1388ad31","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","signature_id":"8b95f05c94191a82a105dd423957340c94a99c1f3d6ef1c8917fd69f25e7d61e","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","signature_id":"d0f8abdb9394d0098892a582f88ae35796d256dfa90792b902dd5844886e4f9b","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230390138736640","occurred_at":"2026-09-22T02:55:57Z","signature_id":"e4f7ed3558fb4fd8a7cd5814747918a0c71dabdb7eba09fe811038658bae7ec6","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":114,"runs_affected":114,"failed_runs":114,"run_impact_percent":16.124469589816126,"anchor_occurrences":114,"after_last_push_count":47,"trend_counts":[0,0,0,11,43,60,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":114,"prior_last_seen_at":"2026-09-26T14:19:35Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":114}],"full_error_samples":["time=2026-09-26T14:49:06.836Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:08.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:09.638Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5\ntime=2026-09-26T14:52:40.545Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/6089940dde784d763a773e4c9004101f74c31b1e2eea1a61564da81e9a0036e5/operationStatuses/08584111727364954260\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 81aead43-b997-4ca0-92b1-5f890a98ae35:9/26/2026 2:50:21 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:40.554Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:41.206Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:42.164Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef\ntime=2026-09-26T15:10:45.962Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/9881912e12f161c343a185ab325eab0f9959440aad9a04d75fdbf5669aaa2bef/operationStatuses/08584111724039322810\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5653504/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5653504\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7053750e-7976-4092-b5c9-05af8bee4bc0:9/26/2026 3:10:24 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:12:45.966Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:12:46.574Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:12:47.251Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5653504%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5c70e5fdb4253270876a8cf87e4ad360a842c5c82a499a0efaf8267c12be42a2\ntime=2026-09-26T15:19:06.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:49:01.802Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:03.910Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:05.531Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84\ntime=2026-09-26T14:52:06.712Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/9ec5d968c4a2abbfa09f2d13ea926004d6d393845d4e1c03095043f158076d84/operationStatuses/08584111727409288089\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3938f6fe-2589-4619-be28-fc66822767aa:9/26/2026 2:50:13 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:54:06.713Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:54:07.745Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:54:09.171Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da\ntime=2026-09-26T15:09:44.252Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/8afa4a83c65198c5a6152d3ce561b308250ec90817107c38ff6b5a156799c6da/operationStatuses/08584111724372557912\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6813568/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6813568\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 381e25a6-705f-4967-8b86-0e72fd08d4dc:9/26/2026 3:09:17 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:11:44.253Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:11:45.072Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:11:46.506Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6813568%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff8da50925da962c46c0feb369b1ccdd3215b84037bcc35598bc1f3f1121f0d5d\ntime=2026-09-26T15:19:01.486Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-26T14:43:15.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:43:16.912Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:43:17.849Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3\ntime=2026-09-26T14:47:48.727Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/553cb6c6b3fded1accbd2852e1223b9f885add89fe9c1ca9b9ab9fd387804af3/operationStatuses/08584111730881613482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 5fd50c6a-1737-4597-a037-368930da6cc9:9/26/2026 2:44:27 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T14:49:48.731Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T14:49:49.490Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T14:49:50.425Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919\ntime=2026-09-26T15:05:54.057Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/99ebb22d91f4358f020049c1b68a81eba2ea522f9bacb97496c9a72de75bf919/operationStatuses/08584111726955290959\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1104128/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1104128\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 169a3df3-f8a3-4c96-9074-128137b133d8:9/26/2026 3:05:31 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-26T15:07:54.062Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-26T15:07:54.711Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-26T15:07:55.697Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1104128%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9b984337a990465b3d1ba4b559dcc1260b91bb6b32219cd0080e1d7d34f722df\ntime=2026-09-26T15:13:14.833Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103852034485653504","occurred_at":"2026-09-26T14:19:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851390936813568","occurred_at":"2026-09-26T14:17:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851378941104128","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851381092782080","occurred_at":"2026-09-26T14:17:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851372238606336","occurred_at":"2026-09-26T14:17:09Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851355876626432","occurred_at":"2026-09-26T14:17:05Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851334296932352","occurred_at":"2026-09-26T14:17:00Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103851316445974528","occurred_at":"2026-09-26T14:16:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848603104579584","occurred_at":"2026-09-26T14:06:07Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848790996815872","occurred_at":"2026-09-26T13:01:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103848898379386880","occurred_at":"2026-09-26T13:00:47Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103790396751482880","occurred_at":"2026-09-26T10:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103789910530985984","occurred_at":"2026-09-26T10:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103786893190631424","occurred_at":"2026-09-26T10:00:44Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103785005913214976","occurred_at":"2026-09-26T09:53:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103775301015179264","occurred_at":"2026-09-26T09:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103774808339648512","occurred_at":"2026-09-26T09:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103764124197457920","occurred_at":"2026-09-26T08:30:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103760196256927744","occurred_at":"2026-09-26T08:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103759712255217664","occurred_at":"2026-09-26T08:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103745097093877760","occurred_at":"2026-09-26T07:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103744610089046016","occurred_at":"2026-09-26T07:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103733651282857984","occurred_at":"2026-09-26T06:29:22Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103730519727411200","occurred_at":"2026-09-26T06:16:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729999365279744","occurred_at":"2026-09-26T06:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103729510443651072","occurred_at":"2026-09-26T06:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714900751683584","occurred_at":"2026-09-26T05:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103714409951006720","occurred_at":"2026-09-26T05:12:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103705397104742400","occurred_at":"2026-09-26T04:37:06Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103701911558426624","occurred_at":"2026-09-26T04:23:15Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699803396378624","occurred_at":"2026-09-26T04:14:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103699317209436160","occurred_at":"2026-09-26T04:12:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103685753023238144","occurred_at":"2026-09-26T03:19:02Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684698914951168","occurred_at":"2026-09-26T03:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103684219090767872","occurred_at":"2026-09-26T03:12:57Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670738761289728","occurred_at":"2026-09-26T02:19:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670723087175680","occurred_at":"2026-09-26T02:19:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670718657990656","occurred_at":"2026-09-26T02:19:18Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670704342831104","occurred_at":"2026-09-26T02:19:14Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670699393552384","occurred_at":"2026-09-26T02:19:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670647581315072","occurred_at":"2026-09-26T02:19:01Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103670621719236608","occurred_at":"2026-09-26T02:18:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667365953671168","occurred_at":"2026-09-26T02:05:58Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103667354889097216","occurred_at":"2026-09-26T02:05:56Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103656984816390144","occurred_at":"2026-09-26T01:24:43Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655450791645184","occurred_at":"2026-09-26T01:18:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103655428259844096","occurred_at":"2026-09-26T01:18:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103654924834312192","occurred_at":"2026-09-26T01:16:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103650148490940416","occurred_at":"2026-09-26T00:57:33Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6775,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641099342974976","occurred_at":"2026-09-26T00:21:36Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641079956901888","occurred_at":"2026-09-26T00:21:31Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641065637548032","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641066266693632","occurred_at":"2026-09-26T00:21:28Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641060163981312","occurred_at":"2026-09-26T00:21:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641026831847424","occurred_at":"2026-09-26T00:21:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641014462844928","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103641015683387392","occurred_at":"2026-09-26T00:21:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103640998813896704","occurred_at":"2026-09-26T00:21:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639551695130624","occurred_at":"2026-09-26T00:15:27Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103639238770692096","occurred_at":"2026-09-26T00:14:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7181,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103632247545401344","occurred_at":"2026-09-25T23:46:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630683309084672","occurred_at":"2026-09-25T23:40:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103630333239889920","occurred_at":"2026-09-25T23:38:49Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103629622468939776","occurred_at":"2026-09-25T23:36:00Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103622374401249280","occurred_at":"2026-09-25T23:07:12Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618033778954240","occurred_at":"2026-09-25T22:49:57Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103618006578892800","occurred_at":"2026-09-25T22:49:50Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612428464951296","occurred_at":"2026-09-25T22:27:40Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7182,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103612411738066944","occurred_at":"2026-09-25T22:27:36Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610246076305408","occurred_at":"2026-09-25T22:19:00Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610217940914176","occurred_at":"2026-09-25T22:18:52Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610216275775488","occurred_at":"2026-09-25T22:18:50Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103610212916137984","occurred_at":"2026-09-25T22:18:48Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103605913662263296","occurred_at":"2026-09-25T22:01:44Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103603168511266816","occurred_at":"2026-09-25T21:50:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103599075277410304","occurred_at":"2026-09-25T21:34:37Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595508659392512","occurred_at":"2026-09-25T21:20:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595507854086144","occurred_at":"2026-09-25T21:20:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595503156465664","occurred_at":"2026-09-25T21:20:25Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595459770585088","occurred_at":"2026-09-25T21:20:15Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595446143291392","occurred_at":"2026-09-25T21:20:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595427583496192","occurred_at":"2026-09-25T21:20:03Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103595282880008192","occurred_at":"2026-09-25T21:19:32Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103594698844147712","occurred_at":"2026-09-25T21:17:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579624884670464","occurred_at":"2026-09-25T20:17:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579604630376448","occurred_at":"2026-09-25T20:17:14Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103575188418334720","occurred_at":"2026-09-25T19:59:42Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7181,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103572043327868928","occurred_at":"2026-09-25T19:47:10Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7180,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103565386573156352","occurred_at":"2026-09-25T19:20:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564555174023168","occurred_at":"2026-09-25T19:17:26Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564545086722048","occurred_at":"2026-09-25T19:17:24Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7174,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564524710793216","occurred_at":"2026-09-25T19:17:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564515059699712","occurred_at":"2026-09-25T19:17:17Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381081762828288","occurred_at":"2026-09-25T07:08:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103380979837046784","occurred_at":"2026-09-25T07:07:59Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7150,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369630306275328","occurred_at":"2026-09-25T06:22:53Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103369316203237376","occurred_at":"2026-09-25T06:21:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7156/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103336399548715008","occurred_at":"2026-09-25T04:10:50Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7156,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318302158753792","occurred_at":"2026-09-25T02:58:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103314299182125056","occurred_at":"2026-09-25T02:43:01Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7155,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305260238901248","occurred_at":"2026-09-25T02:07:06Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103281892986982400","occurred_at":"2026-09-25T00:34:11Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7149,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103277014529085440","occurred_at":"2026-09-25T00:14:51Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269397211648000","occurred_at":"2026-09-24T23:44:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255953611689984","occurred_at":"2026-09-24T22:51:10Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7150,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252721661382656","occurred_at":"2026-09-24T22:38:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103246251251208192","occurred_at":"2026-09-24T22:12:37Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7149,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103232602474287104","occurred_at":"2026-09-24T21:18:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6359,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103213571209957376","occurred_at":"2026-09-24T20:02:45Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103205737139277824","occurred_at":"2026-09-24T19:31:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103198517097926656","occurred_at":"2026-09-24T19:02:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103186015765139456","occurred_at":"2026-09-24T18:13:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103181727441620992","occurred_at":"2026-09-24T17:56:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103178709123993600","occurred_at":"2026-09-24T17:44:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7054,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":106,"runs_affected":68,"failed_runs":68,"run_impact_percent":9.618104667609618,"anchor_occurrences":106,"after_last_push_count":105,"trend_counts":[91,91,0,0,1,0,0,14],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":146,"prior_last_seen_at":"2026-09-25T15:23:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":45},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00172e090\u003e: \n    resource group \"complex-cilium-kv-k9knbpmhbg7v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k9knbpmhbg7v\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n      \"name\": \"4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:43.521207454Z\",\n      \"endTime\": \"2026-09-28T09:54:45.223507829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u15umjtvbfn7or4kh25ftp6mt8ubm still exists (deletion dispatched at 2026-09-28T09:30:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00172e090\u003e: \n    resource group \"complex-cilium-kv-k9knbpmhbg7v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k9knbpmhbg7v\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n      \"name\": \"4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:43.521207454Z\",\n      \"endTime\": \"2026-09-28T09:54:45.223507829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u15umjtvbfn7or4kh25ftp6mt8ubm still exists (deletion dispatched at 2026-09-28T09:30:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002958120\u003e: \n    resource group \"rg-candidate-4-20-n6x2rfcpmr5t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-bkc8nt\" in resourcegroup=\"rg-candidate-4-20-n6x2rfcpmr5t\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n      \"name\": \"4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:44.717303002Z\",\n      \"endTime\": \"2026-09-28T09:54:45.21106923Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ce2mv9aje3edhhstj5ffjhl0mpi still exists (deletion dispatched at 2026-09-28T09:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002958120\u003e: \n    resource group \"rg-candidate-4-20-n6x2rfcpmr5t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-bkc8nt\" in resourcegroup=\"rg-candidate-4-20-n6x2rfcpmr5t\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n      \"name\": \"4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:44.717303002Z\",\n      \"endTime\": \"2026-09-28T09:54:45.21106923Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ce2mv9aje3edhhstj5ffjhl0mpi still exists (deletion dispatched at 2026-09-28T09:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001010288\u003e: \n    resource group \"clusternp128-cr6m72hbhthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cr6m72hbhthl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n      \"name\": \"2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:26:14.153004619Z\",\n      \"endTime\": \"2026-09-28T09:50:15.107483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ep16n5al4fc1ggfac6imb350l still exists (deletion dispatched at 2026-09-28T09:26:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001010288\u003e: \n    resource group \"clusternp128-cr6m72hbhthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cr6m72hbhthl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n      \"name\": \"2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:26:14.153004619Z\",\n      \"endTime\": \"2026-09-28T09:50:15.107483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ep16n5al4fc1ggfac6imb350l still exists (deletion dispatched at 2026-09-28T09:26:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"3e888de17147e7874eadf6ced8362014dc85d0b720dae2bc042d4c0447b893a8","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"41f8565c8d66121e0492feecad4bd386abcae6dbe95e544be1df719574ccf61b","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"538ab841d7da92f151f053745afb8b6c8b0b5a9b56e4c3e14a219af6b04685ad","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"5dfe14507d8b11e66e113439e7fd5e5658e11aa6bef04974f9ae810928fd2cab","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"9750f33ed5e6147a22d180f7699d450888db4a731961f85f26cda84a3169792e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"98ef78430baca42d6f526e7ecca494ec92e181a87375c4f938b1a9850f7a6bab","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"a69c8acf0abd80d165248bcaf55a4ced12cad424d219715d2c11e129770e5945","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"b50a899b542ef0c196a7d92efe90960659f5d0b7a62b38d8e35564b620d606b8","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"c565e2df29978beff1205caf7aa2ba4d94ee0ef7e6719c56612ee152f0096b19","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"e6ee12a3586a43226a4838a0632196b6f3dcfaa9c97b4d56737aff26dc114226","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"edb70c942e30bb17da3821925bed8b5e1b90399f2f5e8e8d2a7f7e8ad2f00f79","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"fe6f233cf035c94775c98365cc42c5ca8d0b415b7ad76008f15194614bdb8ee8","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","signature_id":"b3295d25e69eb7628cbb4ab513ba3757584094fbc4dc76c3f57f93b86af34242","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","signature_id":"d03f3c6aca7be3f6a3c7d8785334ad1db8dd496853f706eb72adae638570308d","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","signature_id":"0cdecef159b757ddf931d683bdcc29ffb06fd65143e02ca7f2cf4ee378941f30","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","signature_id":"675a5eb91c3847f975a012a61453329e1aaa79d3dc276ce4e8bb78f6001a6575","pr_number":6359,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","signature_id":"9ff9c40b0726a271e1572a52f2a2a4d98ad6a5faa3d1535bb48b76b53cdda930","pr_number":7084,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494708675645440","occurred_at":"2026-09-22T20:17:39Z","signature_id":"8a53d3030826034995c1023916f92588c2d2bc0619bf9b3be7d8c7100951a66f","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102494423198732288","occurred_at":"2026-09-22T20:00:29Z","signature_id":"a08178183f0c8eb37b7968df640b81b0fe1b0c10db1e057a60161fdcf72897dc","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102490037739327488","occurred_at":"2026-09-22T19:29:34Z","signature_id":"3c4983ca599806389d740358987cab851287fa208512172337c9a318c08dbfa5","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102479316322357248","occurred_at":"2026-09-22T19:15:19Z","signature_id":"ec2c0c0328932eb53b4b57656bf9a51b204217985c768bbe396defc59424a479","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","signature_id":"909fdafce84c913a896925a9962c4ba8be3902b29c7ec52c177584072fc1104f","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","signature_id":"2c1a5e84b6f1cde3b61bd78d82756fb1fc0ea6de3271eaa74fb9e6ccf95e2230","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","signature_id":"354c4acb28abc19875c6408355327707c18ba1fb5780d90c1a40b6aa98eb5e4c","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","signature_id":"99198eea28d9b68ecc39071e0c0c9a858751bd46d9f111c48b130fa5cb7ea428","pr_number":4878,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","signature_id":"aa1c09867367e81459be0baa88cdeec15ab5c644b3ef04cb4a9c498a3a36962c","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428355495202816","occurred_at":"2026-09-22T16:02:20Z","signature_id":"99cd53e640ecfd1e9759c1c134bdd8a22ad634239849ea0a227a303aa002a83c","pr_number":7067,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102425288674643968","occurred_at":"2026-09-22T15:50:24Z","signature_id":"abf2bef05c9e4545e9170966e3053a83da97d8f9fa426e9e835fa70973be1bc9","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","signature_id":"f8cd22fd3ccc506847bbe331efeae788ed395ca93a5288623e9115e942ee9261","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","signature_id":"454db31ed5c16068e777507735c5d53198406e84bb457480d945fb2e9b36f1e5","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","signature_id":"bea1828c4a4426018e92c0f20fa15d296358e1c6b1f1a42fec58d586a794f48c","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","signature_id":"182f151f4d335c3fa9f666fbf1c8637609ed53fb8725e376854f43c1b072f900","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","signature_id":"6dc54736f3b084b7ef7df392cad229a0edd37b47ab47338769672396e43cf0ff","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","signature_id":"a9df2cc0c774e75bc299fbf77563ccb4e81f10a214bbd561e63e891df2f8e874","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412746732605440","occurred_at":"2026-09-22T15:00:34Z","signature_id":"cf75714f4015843ed8f32a3f4098c758891337ab873b13195fdad1e87c280797","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","signature_id":"82a4812b53d7120071f99657652811f4dd24e212a34a1955fd32863186a418f1","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","signature_id":"e421292d38ce67fa99edcfb49b7aec3948c57668458969aab7ac974ec6685308","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","signature_id":"6b4531564f88abae6232829d157ef359558a3580ebfe6e3dc7e43f0c62215fae","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","signature_id":"70f45997273ebdb95e6e31e8407499ba7b05f852f4b22108605ff4f1ae6eaa76","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","signature_id":"6d3fd822dfa17e31364223c9652ee3d5734e8c3cdb06b0149b5e7838678a62af","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","signature_id":"93314144e1cea207b5ebb0b54cca0d2401a84bfcc54f2f1a3b60890f1aaeaa3d","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","signature_id":"be55dffe861468905d6cf5bbb952f4afc7acde7b16ebb5829460c64a53913043","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408218671058944","occurred_at":"2026-09-22T14:42:34Z","signature_id":"f705eec296baf9972cc07d4901b687015d1dc5a25a5effbd97e4ccc48c8a16f4","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102407137983467520","occurred_at":"2026-09-22T14:38:17Z","signature_id":"39bd8a304f7e6731ee8ba3d539f47311cc9b577598f4fc5464828fc4148fc7fa","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102400420944023552","occurred_at":"2026-09-22T14:11:35Z","signature_id":"0c4f54710868a82d6cd4ab188b8e5b92ccdc663bd7044f1d176549c7a1679554","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102399637812940800","occurred_at":"2026-09-22T14:08:23Z","signature_id":"501591f861d7f3032330c71c8a8110b35bd8c46439dfa6416ccd3c8fa3fc9d55","pr_number":7072,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","signature_id":"7e9d178e6c594517bad8d5499ee753b7577341b05fba4f102298178f01197cde","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","signature_id":"bd2d83ef948cc00c872335719fff75d3ab902eb5b2a91c73605693d9a4d11d71","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","signature_id":"ce439a01f3dd078882fa3337ada81f946640cfb68bece27f3dc970d7a2c530b0","pr_number":7001,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","signature_id":"3aef474821203f0513acbdb2bd4f218706dfce0520096977bbd87f680e92cc40","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382907266961408","occurred_at":"2026-09-22T13:02:00Z","signature_id":"27062e67a1d88224902adb713a26696bb6c66d8e52b518684911402a1e0836d3","pr_number":7067,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382852271247360","occurred_at":"2026-09-22T13:01:46Z","signature_id":"be110f491d5014ebfa7f9314a49d9edae412a9f9b47e864700f474e4f5c5ebce","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","signature_id":"b368b19d396235037e1a94307d4bc5fed0b37f15b10f9fdb4873cee59285f0a0","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","signature_id":"34937e81704d7c1a456562c7e0f20bef69ca74160d4aa8539e4e3d2222a51ffc","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","signature_id":"928a4c5442ab7d0c4e8b9b1125e98edcd3463091d981041cf5ac778d3757533d","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367395485061120","occurred_at":"2026-09-22T12:00:21Z","signature_id":"97f5e8b0ca0a4de77ab6536bf9a3b7de05d88fdb06f441d1fed55e967b55be5b","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","signature_id":"9cd2d41b80cce088eec466c6b8d8c74e6008102a92ae17f83120148590a9efe2","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102359588845129728","occurred_at":"2026-09-22T11:29:20Z","signature_id":"7c852bc88bc03a1f6a63c0506c2a307d43133205030b3cd74f906a160ee3d56b","pr_number":7070,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","signature_id":"5626fe8511ed4c208a2b8f6e8fcb335eaa8a9082722db26ba8fcf76ecfb81810","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","signature_id":"b0c4616901a6dc97e982f9527c5d28cbd309863e5269f862500342c9b8619681","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102353447767511040","occurred_at":"2026-09-22T11:04:56Z","signature_id":"11a0a2dbbb69ddfa8609fe29df3b8e40cfdda6cce08442acfb43cebd7b72a660","pr_number":6092,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","signature_id":"01316794d9c0fb344341457a3d1310352434baeccefcfd1e361d50968abff590","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","signature_id":"c5b55b9526fcca16eb255c0896e3a68835960f08ba55ea57d84d15dc778d8456","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","signature_id":"d0f379246b4a2c91308e6d2f9e47ae91e7bb3d786e3223b6c8141724a6fb1201","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102352929464782848","occurred_at":"2026-09-22T11:02:52Z","signature_id":"e9fae8337be48ee3b5d096071bada7dbec389c7f5d2bbfed662ea298bb36fe32","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7067/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102340054952185856","occurred_at":"2026-09-22T10:11:43Z","signature_id":"4afad7a628492d65a31b44f111c2f91a70c0ac311120930e8d36551c21514ccb","pr_number":7067,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"320ab412a4e2804f8ccd6d79e9bdb48fcd873437f28371c785640198716dfcc6","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337461471416320","occurred_at":"2026-09-22T10:01:00Z","signature_id":"8d08eca2a140bf94fe932d5a0d7fe9fea9199044029dcea39bb14577310f4851","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336356570107904","occurred_at":"2026-09-22T09:57:01Z","signature_id":"db02d20b216e98887346005e3d003ac57a48a9edfe9a37448a2d26d11e0bffc6","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322611546492928","occurred_at":"2026-09-22T09:02:24Z","signature_id":"b72d73aed22f5ea9b8a850bcf6cbcbb6a47fcdcfbb032249b5650e114d6660b6","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","signature_id":"e67baa0dd3ecf57ed16b62b6102e97d4b4b5f1d7f193c304c25d15049f8d2d1d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","signature_id":"68971a5f68fb618afb67d5665d57f00e7f0ad4c7b3d52339c1dca509cf1867fd","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102307266211352576","occurred_at":"2026-09-22T08:00:53Z","signature_id":"57f6ec7707b46ec3fc5d943d1acba1050e14974e279203d5fb4268374fcc3f88","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","signature_id":"1eddc05aa88f49be251be96845a2a982e308ac66e6019a984d6ae05ad8c0e7a6","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","signature_id":"4d3127d69b3237c6f0f072b82641079832ba20fe44b9241738bc8153a3a12cbc","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","signature_id":"e6d21a6a368ee63de4c6cdddfa3fe07749c51b83e183aac5c0c8f26cff689fee","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","signature_id":"eec6e687779bd9082c44d49d3a20813b73f9a8c303884772641366c4bd373d06","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288391285510144","occurred_at":"2026-09-22T06:46:25Z","signature_id":"0cdd11030bef97770abbc84eb63e790e0a2d96f7178f49901404981b327067e2","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","signature_id":"47ebdfd44bc097708d001bb4432cee3e0d62f312c88e513f44ba0350a0c2b83e","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102288344015704064","occurred_at":"2026-09-22T06:46:14Z","signature_id":"de54a79a3cfc986fa81033de7f4ed2ecec96374717c110bb677c9614805470bd","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","signature_id":"da19aec62f53374613dfa57e08b8efed775a8884eb19a2bb32427dac86c41c5c","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","signature_id":"ae65cebd1e31dcea26a17fc935f9947f5012c586a63c60d1a7132f4c0742cf50","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","signature_id":"fd972711a887999b78faa8f83122475bc265f258ed4abf1613070a868375c0e3","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","signature_id":"428da8905e3f807461473c236e1fbd94195470e327e2f10f57443086600ef92c","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","signature_id":"c7ab45585cce0b2cd1a21ce1fe6993c7311b39d4cfa42e2e851f63f05e62830d","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276703479926784","occurred_at":"2026-09-22T05:59:59Z","signature_id":"620a806bdf69f493c7cd866ed882325652032b150b55c86a742a8b7ce5cc719b","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276703479926784","occurred_at":"2026-09-22T05:59:59Z","signature_id":"7f12d0e1a62bedb37c8fe232dd318ca8e10bd4f5442c8d1539601e78b02c5186","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","signature_id":"3f097e8045967cf6f7e9794619ffbd78d0b906f1fbf45cdca79e33043cc7c05c","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","signature_id":"6bba268d58253a958441751f28c8663ebcc280dd195bfb45f8bdd67ce4526108","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102272452842229760","occurred_at":"2026-09-22T05:43:05Z","signature_id":"8f76fd8eb82fbb3781424e84e450beb330d864e7447398473fd453d51af7bd5a","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","signature_id":"f9be14c948f61ac7c219c13da423855cfb6ba92c1447de4905c30e0f01392252","pr_number":7060,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102236197437837312","occurred_at":"2026-09-22T03:19:01Z","signature_id":"c126cf9868b6fa1682c98b7ec463ce556108063b12012548fba4b2cbd50c1637","pr_number":6891,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231619598290944","occurred_at":"2026-09-22T03:00:43Z","signature_id":"48c73f0a820a7005cd785a644d1be18a505b1fc1bc6e813c6a8d92422eec2cfd","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","signature_id":"0704ec53df87d0f6d61cda9e5060a6b25d6066281db8286b2f544f0bb012d2b5","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","signature_id":"3ac4aba76510c25db23049b52ca845fdda949e4eb97a61d8fe42b96c02253216","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","signature_id":"71286f605bf6e11a92c15b43ed0ece7a96a19a4336765536c2ae052a35632578","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","signature_id":"ad7859705e4441294e9606a8e1e2039a0669249b9198f60c803017117efd2c42","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230542270337024","occurred_at":"2026-09-22T02:56:33Z","signature_id":"8659fb6913fb2dfef84644160b413f5872d6566f70a0cee56ca0e1618ca42d3e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102230390138736640","occurred_at":"2026-09-22T02:55:57Z","signature_id":"75046b60e1f2f39e795e5adb5230e5ac2986a3bec4514e7d4669a5e735bbaada","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","signature_id":"12855f5c94d8ce8642a8edba1fdb0e1255692259e41e52b94faff68410c3c55b","pr_number":7060,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","signature_id":"a8ff04f71f30ac8836925032560c4015403e6d160d584e18c3135ac999e36302","pr_number":7060,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7061/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102203845328244736","occurred_at":"2026-09-22T01:10:26Z","signature_id":"12732a5ca59d07fe7708e2bc28f4d0a32b1968e01ba973942f97caff43c12fc7","pr_number":7061,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102191495971868672","occurred_at":"2026-09-22T00:21:24Z","signature_id":"4d055ed76535ebb1f15e7aca5a47519d6e1a265cb5ef2d99a239a68927a314ab","pr_number":7059,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186809273356288","occurred_at":"2026-09-22T00:02:45Z","signature_id":"c801a40928cf7ce27dc4dc008fa87ca8f70bc430ef051fbe1974207aae546b74","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","signature_id":"60822c0ff2befe11e2e3b0e3ca8a5d85423c4687184b8ad104d0ffbc18cd2b02","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","signature_id":"e1f75fdde1658fe53d9052beb10bdbedb5f8af112cd433dcfd1ac576c7fd2884","pr_number":6945,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":77,"runs_affected":77,"failed_runs":77,"run_impact_percent":10.891089108910892,"anchor_occurrences":77,"after_last_push_count":44,"trend_counts":[17,31,18,28,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":107,"prior_last_seen_at":"2026-09-24T16:09:16Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":77}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T18:08:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4417536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2fbf2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-2fbf2 (namespace arobit) on cluster ci01-j4417536-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:41:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1914624-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8vzwj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8vzwj (namespace arobit) on cluster ci01-j1914624-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:49:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1279232-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5zn5h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-5zn5h (namespace arobit) on cluster ci01-j1279232-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","signature_id":"66131251b5fecbeb3371e744344966e17e34b09c66d06812a5e0fe0811e83ed1","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"4e2ec19ff9d069c5bb64045806ef329a6aca9db297b07736e4b201ead6847e07","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103145551821279232","occurred_at":"2026-09-24T15:32:28Z","signature_id":"8c94a4b03784c995aa979e22c654512f8c6707bcd9d456af76a36f2b9dd1e313","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103139552272060416","occurred_at":"2026-09-24T15:08:38Z","signature_id":"2d8869925bade65a292b27f5efb8dda6ad2b9c0f4a99be4072145040c21d0bd5","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"ec42ed0291695c354e24001557eb7c5d81b9f3854081433267b5026166de670e","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103114369016532992","occurred_at":"2026-09-24T13:28:34Z","signature_id":"017b8f139e649921e31c93332e3a78fedbb3f1194cc2d5d6b68bc998ac21643b","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","signature_id":"db67d276ff0371dcc22947883bb69cbbfdb27c1787f10ba7322cfaa3b73aa198","pr_number":7131,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103106328183443456","occurred_at":"2026-09-24T12:56:36Z","signature_id":"03aacfb93bf8d8b6783be4fea0e1b48e62869181b58bb5f7f4cdf2c172610e39","pr_number":7132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099181529829376","occurred_at":"2026-09-24T12:28:13Z","signature_id":"ae74f769fa6d33550badb8679083a5b953a0ef29b38b87aaf9b9ee5f82f844cd","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099182641319936","occurred_at":"2026-09-24T12:28:13Z","signature_id":"598f96ff4cc7f38b9158674f984f82130937ad37ec6425753f1589dfdb918be5","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","signature_id":"ac2d659b6d8c225f448dfa59b70e7e1449c341473cabe70fbc8cfa0120d5269c","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"c3c8df3e62c73e7b4c4be31e24a070ec366f9237969b5d0bdd9fb689ac18eaa2","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055146278719488","occurred_at":"2026-09-24T09:33:14Z","signature_id":"a410a024ffc0f26ed41523481fe4553235341e68f41df89e12a0df8d70a45e2c","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"fecdb95305d7912b34e85ac60e3a333a4105e66f05be7ed59027fc3150684b84","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055124225069056","occurred_at":"2026-09-24T09:33:09Z","signature_id":"94cc4e8f43733aad08243e0c9d7614fcd959bf692829e56a21d66324fe6568f8","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055116536909824","occurred_at":"2026-09-24T09:33:07Z","signature_id":"33cdaea8b871495d558c36e8c9add34f9b20b7366192a7363dd1ee244293bb4b","pr_number":7127,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103034733737807872","occurred_at":"2026-09-24T08:12:03Z","signature_id":"34737877aa3d834f2668fa205d349b5946f34479dc5d4c773b3f919ae25587c8","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","signature_id":"967796441e8b38c863ade49bdd491e7af8db03f90c559c543dbfafc769074a03","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"774298937099e086350da5d45672b54735082f9b7780d4691b1ecf885eb8f21d","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"10f8755438a83f36559eb45ee70fb70663d5582f7bec31700fca4ba57353d6f8","pr_number":7126,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103001507350712320","occurred_at":"2026-09-24T06:00:05Z","signature_id":"6fd93723b17ab024ac620ec98b9e4488b425d7110afc72440506d1c61f3eab4d","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"5417773a0fcbc7245cd61bb18c7acb92d76ae842a3f206a9a2b3bef948abce1f","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102961799233015808","occurred_at":"2026-09-24T03:22:14Z","signature_id":"ce1c532ec2e46cd04d99a6539b8974a73f6fa6a9c1f96c6db3d23304ad9e593b","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"3797bbedc99fc8f57829a65247ea354a468bfe71a528c5bc92588f0d1aa0858f","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916121228218368","occurred_at":"2026-09-24T00:20:48Z","signature_id":"87d3d8dba8e6534afffc2d80f759bd53c5872f8a0e40478e7a7531c5b2e7dae9","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102914094729269248","occurred_at":"2026-09-24T00:12:45Z","signature_id":"79341176e7e5e053b430da83424bffbe33d8cbf6966dd23ee9a8f0cd6624d271","pr_number":7123,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"739f026725e19a96d09fe6b2030f5896718a89453fad23e329ee1552ca883a94","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102912928209440768","occurred_at":"2026-09-24T00:08:06Z","signature_id":"6e5e5e7d518f60d91ecb09219367313b7d859bc18594e43d3b6d1d6226085c18","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"45c5b936de9aef65cab348b5a9c37cd387b152250ed997df53a6e8778dabfb08","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","occurred_at":"2026-09-23T21:27:07Z","signature_id":"aef4de3c1d8c7e8e67123da5e644cdc4d14dd0dceea99369d4b941edf923b506","pr_number":7123,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","occurred_at":"2026-09-23T21:24:41Z","signature_id":"d71eb8d3e94b0946ce9d45150994a79c1da31f720176e1cb83633088dcca59fb","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","occurred_at":"2026-09-23T20:49:31Z","signature_id":"a051597c46d9f96e8b20b9310e059e4f799849ba3ec132c05941edb67268ea29","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","signature_id":"254dd39e2f0e4cec1b71ce202f610ab07b9da883c1b746d73e09e4f13421506a","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","occurred_at":"2026-09-23T19:19:30Z","signature_id":"8d0ffdcc532373f83199e900f86943923bf9464de78da96a0bc81c5d10748030","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","signature_id":"f01e8023db70ac4eb379611e2c78c4d54f1aeadca78f25f3826d0ffec54edaa2","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","signature_id":"312cea2b73363d1b160f9cce1d956a42a926ffe6d5a41922d416b5287eec53f4","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"cedc29dd595c13bb38b5f8171952695156bf1b2712e54cef49d283d640a3a08e","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","occurred_at":"2026-09-23T16:43:09Z","signature_id":"f1b24a151cda52cd0d54ea6a4677f8fb34ef83c5bdadd527eb334c9d72e78bde","pr_number":6925,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","occurred_at":"2026-09-23T16:20:32Z","signature_id":"c0de1a0ccf1f7966d446e232f88e4d90f26f4017b261cdeb15d15f7eb77aa118","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","occurred_at":"2026-09-23T14:18:47Z","signature_id":"df2c8e2eea124fae55058fbdd964feff88c2979d18d038ec6b6702d5cd247db8","pr_number":6687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","occurred_at":"2026-09-23T13:26:28Z","signature_id":"27fba5395347858a8e8e715d32f7f0fe022ff7065fe1ea71ae9a42e0ed9a0a0d","pr_number":7044,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"c5b74c40552ae1b36fab99026dc41a2a915f1d1a792b500821eca9610ee3d31e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"b94154bec553ae4d7de90c235afb929dc5f8241123d4ff71acb4b324e3664bec","pr_number":7103,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","occurred_at":"2026-09-23T08:02:34Z","signature_id":"6a553ba910a4f3a893c5726596a17745a39160fc0a64f583a6cc5c05a5a4ea9e","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"61fa061764ad73e8317dc73f0ca153b35f59fa073234e6998c92f96b627530c7","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","signature_id":"398824eec9025702d224d5a9ba36e64c2703dfdc5e00ba0eb6afbc886a9d202d","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534129747759104","occurred_at":"2026-09-22T23:02:54Z","signature_id":"90465e28739c8b20aca20c879dc28759613fff25e486dc294036702bbd11ac8c","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102490037739327488","occurred_at":"2026-09-22T19:29:34Z","signature_id":"32f305375aae0b604305ebd3d7bb6bc5f6c89d67ce915ba6ee48e5a7b097541f","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","signature_id":"251163f0fa415413d3fc13e342aa1d4c5b3c3761b29974ac0d10e823d63b76d3","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102458797258706944","occurred_at":"2026-09-22T17:48:09Z","signature_id":"ab1804d22b808f7b28e0e2b5c1233de332584f8e97e10dfaf11d22bf3d5d0c59","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","signature_id":"74c245df6880b6c37600b0c6bbfe6a2573fd5ffd089e497ae36eca834db88eb6","pr_number":4878,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","signature_id":"04b8fb1a73deff541d6dac26b2975c6beebcaf18714f64bb6b47b58d5279c5eb","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","signature_id":"47de9b40c36841857354a6bfe3c0daa443eefe7d83e1b1fd6b65637d19ada076","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"72e6ccb9b356ddbe42696b63bd109e76bb2f91fafa56b87cf5cdf1bfa7e627b3","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","signature_id":"2691f57bdab8f983b45857d3645b58fef98f444d625e7e014189bdfd9a9e4e78","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","signature_id":"df11f48db0c6021e63f67dc6d870c79b45967a8dbc153562f9b9af292144b654","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412694500937728","occurred_at":"2026-09-22T15:00:21Z","signature_id":"98654b9625183e68d8396ec4dc88e8d3aa8faa32a967d767bdc548bf01023385","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","signature_id":"83ce9eba8fd7c4927e958e70e9923c68764adb43fd3779794c7181e2062c4a07","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102399637812940800","occurred_at":"2026-09-22T14:08:23Z","signature_id":"0b0c62094340da20a3f28022dd2ae1bbdc1bcfdc0b4bcf968fa111ba233899d0","pr_number":7072,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102397479768035328","occurred_at":"2026-09-22T13:59:53Z","signature_id":"a0369e4b4747ddf65f74013118f1b2acd9e1cd85149284da363bc09e58ad009b","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102388132350529536","occurred_at":"2026-09-22T13:22:45Z","signature_id":"42ed4dd43bfd0f010d040aeeeb49ff87ef925441b0a50891a0c9f45eef3a169e","pr_number":7001,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","signature_id":"c7981f65a6291a8ebee62a5b85aa3f6972edca569a1b1203b0c8479137b02024","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","signature_id":"a775ad6559303e11abdc6f06a0ac66c193334df7354f421e2e170c6c4c7e9368","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367426107674624","occurred_at":"2026-09-22T12:00:29Z","signature_id":"ff9223586b6f06f638cd8392395aefcf5740b8c5345bdec7a43fdef9d217a7f4","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102359588845129728","occurred_at":"2026-09-22T11:29:20Z","signature_id":"3491f500d38d72a5dade4517718158749b870cd35afca4fd6f9298720307de7d","pr_number":7070,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102358117298737152","occurred_at":"2026-09-22T11:23:29Z","signature_id":"32d007958a1c603d42d30427965ccdcaec85034f2288cd66f62776e68fbf4cac","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"cefa57c82ab99c9bb1b4362ad54a67403b423ec2ce56cd2a82667fd307603621","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102336356570107904","occurred_at":"2026-09-22T09:57:01Z","signature_id":"a73ccba8a620f8479e904c1b860b84849b279a7a78182d97426a6810dde93c5a","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102324260725854208","occurred_at":"2026-09-22T09:08:57Z","signature_id":"c04ac85d28af75dbac28d615edf81c669fe6d30a818360aeab15677cb35e9184","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322611546492928","occurred_at":"2026-09-22T09:02:24Z","signature_id":"0eeb5b4008847244d310d0b89d87d8574f8d15776d3de636dae9d610aa73bc6b","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322580831604736","occurred_at":"2026-09-22T09:02:16Z","signature_id":"511274e848bdfa86676025b44ef9d040a781c41c25c3a64f18cffbd4ffd6791d","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","signature_id":"450bab3ad58a1721f7009c09abe7807541a8c718e9c8576a60a01ca17fd9c0eb","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6891/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102236197437837312","occurred_at":"2026-09-22T03:19:01Z","signature_id":"eb32a26deada53d3c0554c1275d606596407420af9fc6a315ca6a0a9fa72f5a5","pr_number":6891,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","signature_id":"7b40679fed1158a25c62a0c7e6a9546444082fb4c154448640ebd2d79aa45c65","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","signature_id":"f890e54a3771983140c6e7e468f8358fcb9025cd81792884f77a2b7344615b44","pr_number":7060,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102191495971868672","occurred_at":"2026-09-22T00:21:24Z","signature_id":"23b506fc45f1e15d2a4b76c5499af25f7494f8b605bccf9085abe7a28964c72e","pr_number":7059,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186737458483200","occurred_at":"2026-09-22T00:02:29Z","signature_id":"0d5065ea065f8b5d5b038c244627dc339027669eafe53ebeb82258c8b157738e","pr_number":6945,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6a7d547f315b8e87584e84792466ecbd2df203b16a9060afa60645bd473178ce","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":56,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":56,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,56,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T12:46:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d58a20\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g7q8vnwmjvvk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n      \"name\": \"ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T14:06:37.339670093Z\",\n      \"endTime\": \"2026-09-22T14:35:38.595765601Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d58a20\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-g7q8vnwmjvvk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n      \"name\": \"ea0270f1-32ca-4b4d-a131-126d53d2800a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T14:06:37.339670093Z\",\n      \"endTime\": \"2026-09-22T14:35:38.595765601Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c0b80\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qwx5tf2dtpk2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n      \"name\": \"5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T13:32:01.635132061Z\",\n      \"endTime\": \"2026-09-22T14:01:04.903065926Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c0b80\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-qwx5tf2dtpk2\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n      \"name\": \"5e31a2de-3b83-48d4-a41a-2116a2e676ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T13:32:01.635132061Z\",\n      \"endTime\": \"2026-09-22T14:01:04.903065926Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-kbjcpq\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f728e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-kbjcpq: failed waiting for cluster=\"cluster-zstream-4-21-kbjcpq\" in resourcegroup=\"rg-zstream-upgrade-4-21-ftfjp89m25kk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n      \"name\": \"3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T13:32:22.061338875Z\",\n      \"endTime\": \"2026-09-22T14:01:24.909726517Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-kbjcpq\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f728e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-kbjcpq: failed waiting for cluster=\"cluster-zstream-4-21-kbjcpq\" in resourcegroup=\"rg-zstream-upgrade-4-21-ftfjp89m25kk\" to finish creating: GET https://rp.j7129984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n      \"name\": \"3d11e0bc-f5d8-4182-a6e1-387ef6e21ad9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T13:32:22.061338875Z\",\n      \"endTime\": \"2026-09-22T14:01:24.909726517Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"05bb22ed0e241c1167007404c8d4e3b91da97b4772f7ce448b2d4b48caf1c2a6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"0f3709ad519f7e7b8e8abc75a8dee44b2ab12631351203d1466b075bd3f2b05c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"1077ec918010bde6d0e63b9e0cb832071c18f4ca2227a1a6e2444c0f54217012","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"125d7b2a1e64c2c36598f3d8585ea4489c6478937c146dfc9a85d370146b92da","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"1a5d886d4d80c8ab33aeab161d42302f0898e33e7b32b74d3ae3ffecb4fdaf49","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"1b7078ed292312c9ffe8816194aaa85647b1c2e4bfcf09a97c4dadbd137c675b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"1d43ccfbc61f660474c97dcd9a29831dbbb576bcd4d61281856d22526f3e3dc2","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"22aedc52ca836cdf7d514d3fba3d71461df6de6eb162c1dcba0ac4544196e286","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"25e161c3cabcd69470473f37e02d3636693f8520b93e53149858575c5f7ca818","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"26bf64aa34ca1abd16a8823ecf0cbcbcaee37cb43a5cfe37931577af830a986f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"27132487c8d28776a72cc37eed74057cd528324d345f115013efb32ee9878f10","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"2bdf2d474187f006f2f6858ad11138f1aca63f2cf4d0bc3da3af6070e6381cf5","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"35442fafa4814db9dadf16a96800b231c95dba26154f748a821779c1c450c5ca","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"3cb932c0e4265483c436c655e1b30c54e08b393cc31eecbbc2f5bc67afd2613b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"3f191fc29b9245b9c91f145004f0e87445cff99d866e034bacd2dc82342ffd6c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"41f4c527fb82523b18c53454eebee84e13213799c50b84ac6f5104114806a006","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"44ebe74846f27c338b0020b629b93cac4c94febab64c3e4ea5e2dc4c13ac6911","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"4c9febb5685c00820373c671b942a0ad6d4af8fb290a59e20e05e9220aa6393c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"4e6005083fa58df0062477155be492344adead24679f90800563c50dcb10e3e1","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"4ea5bdfd0c0fe13c83e36ea93cb4d6b05660b725346e80b5ea45e3584fde7ecb","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"5c546180824a9822a969b1cdd8466c19f6ac94f9341bc046d0d0d71b44d017a2","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"62c72b9790eeaa622371cd4fcce3f1a55c9e869af1565ae1854130331bf7ee17","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"64d81e706a219ab0e795d214a529e53451bd79f2cae0bb71190437d2d9a85d3f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"67667a59b4a551350f8d79402114e1c840bbf22a863b684a641fd345472b2e4c","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"702eb05f62f599c9db1b0b8e344c4e364171cafc8fecb8a2a28b47a1b9dcb72e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"736b4003f514984286a44fd82ba86089bf1764ca185c1772eeac867d501348c1","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"75c907116ecaab8397397d390ad70c1823faa982df325a591793d7d6259ebb46","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"7b61254d3b2425439673bf81a0dda06b0179ac653363cac4b7966dce8a437e5f","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"7f08de17fd820b423433af5f14d7253b852e4fb89158f413aa90ffd6c09047c8","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"8008fba97339baa5071274c98aa0e725ca29436c7f8afc131f7de5f1e67df6e0","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"85afb00f76eebeafd5afd1181f0b8eccdf5062283ecc7bfc786cd8040de4cf44","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"87092b0de71dc07b55bc06c22f40f3a2eb74e067b0845903e93faecb3fea9761","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"91d76ee6d298962789879c47d4545a4200ff5cbefc6f9bc106201b6ae385dd8b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"945dba197673b6f485c758b73f9756c8272cd1cff7d79dfa0c33b87db2f91428","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"96bea114bff0b91e68ec24f631e1236819bf685d8d5765c52b8154dc1a26568b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"9766ef3fa70c1671f3802b255d196eba0c78634f0e552de05e0ed3a4c8949254","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"97a6d366d3affe44798b8e6531d0e15d83c56eb0785ae18b4d157c51025dce96","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"a24c34c210d135aa0d6e6ff7c3d0f3267f00c546b3e06d7da5efcbd1b4502e66","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"b03bb3905da0a0d584082f97b972b91c8e4e3cddf5cf41ebfe6b43d7c5c61e53","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"b31be570968d5cc95f200746d9e769721c920835a846af0f9e792531b00e0ac5","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"bc965970f4ca24037e7e7ce859c4604c1f5f11358722e75e0aec3f0a8e4c60fa","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"be58d2e7a0303b1f567347f14f904f74f000e733efb20ae68e901f4e2a588a20","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"c412aaa9e3b24cac733542e008d37a90317f70a3ea34642e1c14406364c1ee06","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"c42e87688be4015e90e55364d94e2cfdf70fc7be2f1df40b5b9a613da178bd5b","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"ccd547377a6a9f24e00759530e1b5da0ca0add6799d2896e9d0d48f1085974f1","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"d33a734fa6fa20e8c750af55af3edc29a07a36d78fad26f156f3236f9dcd07a6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"d5dbf4b895cbcceda926d5e609a4c7ccd9fd21955e94c103a607056e3c975392","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"d6a3a01776dbf01fafbc37aa5433cf753bfef466e178054f8f54f9b28b7343ba","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"db2c996d9ab000ec5408777eab57f880c1c9caaa8695141c93132b0072cbbeba","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"e08c3b837f2d8a64eba9a61bf8290a329e4e4689019bc4c9c513e0eb9c9e8227","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"e3bd4a02efe99f891912192a8188490059f0792d474be03435f8296210fbe5ae","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"e5241bfbb33f7be045964d3491b282166f16778f8ac1f32bdf2e605f8ad176d7","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"eb1eec2d9d98e544944ca5331858c8445c1124a22d49b02bcbddc76788729d0a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"f22aceb749f662a2e174bb580556604682086f921403ec06374941a12db33605","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"f39fe90d35e76033ec3a3f6c1f03513b571ac19cd4414444db52b5e759592585","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"faf4c3955df8db29a640a24fb0d0dc9027282375fadc9d070d72fce2067bc0b9","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":34,"runs_affected":29,"failed_runs":29,"run_impact_percent":4.101838755304102,"anchor_occurrences":34,"after_last_push_count":14,"also_in":["INT","PROD","STG"],"trend_counts":[0,0,0,0,6,2,7,19],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":29,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86d20\u003e: \n    failed to create NodePool np-m8jhjs: failed waiting for nodepool=\"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\" in resourcegroup=\"rg-candidate-4-23-f6rffb7vgk7k\" to finish creating: GET https://rp.j5063808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\",\n      \"name\": \"451c3970-ee1e-443a-8418-d00eafee945e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:03:14.031479379Z\",\n      \"endTime\": \"2026-09-29T00:22:18.567634171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f86d20\u003e: \n    failed to create NodePool np-m8jhjs: failed waiting for nodepool=\"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\" in resourcegroup=\"rg-candidate-4-23-f6rffb7vgk7k\" to finish creating: GET https://rp.j5063808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\",\n      \"name\": \"451c3970-ee1e-443a-8418-d00eafee945e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:03:14.031479379Z\",\n      \"endTime\": \"2026-09-29T00:22:18.567634171Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b234e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nl2nr6\" in resourcegroup=\"rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\",\n      \"name\": \"af93dcf2-82e0-4298-856d-f4846917b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:46:23.744577199Z\",\n      \"endTime\": \"2026-09-29T00:05:30.871348708Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b234e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nl2nr6\" in resourcegroup=\"rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\",\n      \"name\": \"af93dcf2-82e0-4298-856d-f4846917b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:46:23.744577199Z\",\n      \"endTime\": \"2026-09-29T00:05:30.871348708Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff7440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rdwv792csbm5\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n      \"name\": \"bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:36:16.368959757Z\",\n      \"endTime\": \"2026-09-28T23:55:24.990728156Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/virtualMachines/b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-d9rrr: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-rdwv792csbm5--managed/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/availabilitySets/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff7440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rdwv792csbm5\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n      \"name\": \"bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:36:16.368959757Z\",\n      \"endTime\": \"2026-09-28T23:55:24.990728156Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/virtualMachines/b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-d9rrr: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-rdwv792csbm5--managed/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/availabilitySets/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709786435063808","occurred_at":"2026-09-28T23:08:11Z","signature_id":"ca1052bcf33487a021334893241ae8188017185f1642b6a294d2994f137f4da1","pr_number":7233,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","signature_id":"7a7e77c690889a19cb56e1f50e820d8f0f557b662a6c09c58c3793ba99c49cd2","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","signature_id":"98e7af2c661ff33cf1fa2bc30a96c5bfa7c31b06b9aa1b278fa59c5194434907","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"0b09d572458a381dac48ffb0eb837c9df3852335997b0c4dc2a876b33b08a4d9","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605288744095744","occurred_at":"2026-09-28T16:12:53Z","signature_id":"75a042124b3e99ba945259807210552155a890c136bb34b4fa60646036997e91","pr_number":7221,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104591950119178240","occurred_at":"2026-09-28T15:19:56Z","signature_id":"72df9d9a52679f65dde671993fc72f0a08f84ed93d624648a2a5660cfb58bcff","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104589634188087296","occurred_at":"2026-09-28T15:10:44Z","signature_id":"e7ecb159e629a80b264d02c522765675f076609f52edeb1aa4e53b5b36a030c5","pr_number":7185,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551522074365952","occurred_at":"2026-09-28T12:39:18Z","signature_id":"c24e3d212804f8897ef2c81ade146070e7a5fd62032965fa1424ca72aa49afbd","pr_number":7221,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","signature_id":"d72030c4a0f636f0bfa24f277e9369a539321fccdfff48a303919b0b450bc268","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500979540955136","occurred_at":"2026-09-28T09:18:22Z","signature_id":"4205a7cb8605e3a20406441c549ad2d3c1b449fed823b20a747976cb9404f3ee","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"8407ced857bcce3a5f161f49e7fb898405968e74316d0c49111a8c01a8f61c87","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441018513362944","occurred_at":"2026-09-28T05:20:12Z","signature_id":"5066cec9d4455e60a98c73837fba1de819802d265c19a56ae10aab27b4f06354","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","signature_id":"ffaedd37a7cabdde88484871ed63e277dfa5e461f7bbb1806cffb14d8667ee33","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","signature_id":"f24501ad0818be8a83c15e2425a5d1f36053df6413c3f821edd665347627ddd5","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","signature_id":"05319646bad3ee1f498711d0d9a69ef24065fb55d47f09c96176ab2ba5be4374","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104378704682553344","occurred_at":"2026-09-28T01:12:35Z","signature_id":"5ffe709affd5c890fbd83d8b5aad5337b8d1b174da24da2cf78360ba40199541","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","signature_id":"1849503c0698075db16bbbbedb636abe0bb4ab72f8a7bba7219ce1be3d1d71df","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","signature_id":"11301c96eb4baf025fdfd7d58c7dd82d6896e42f11c7ef2cc1e150fbfe5313df","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","signature_id":"ab546a099515b9c90776a9116c547ec830827b9274733c363741cfc037e250fc","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"01e7808196f8b35484085dcce2391cd7ff226b123e9d686fab532297c48cc3ba","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"0edcc1c21b78bd45a5dcd2f988466a7ec1ce2a2761e3091d1504343efd9bb710","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291389238939648","occurred_at":"2026-09-27T19:25:37Z","signature_id":"eea917c119fb496a73e78c0ccb01ffe97195fb9bbf05312d263c9d161c01f38e","pr_number":7201,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"0263dc178beb748cec45d68667416149811aa14e2aff4db55448a33e6e63b5c7","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"0ef5b19fd1cdd2f1706442a99609b4839aa21fe049bf6ef3221a44afe9300590","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104116344919166976","occurred_at":"2026-09-27T07:50:03Z","signature_id":"f11b81cb85742fe5ff7b9b961ba857992240995905349698dcbcbd8852080bc6","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"673f5d1216f0f7b060fd38c36ae28fe2306c5398d3ffd8fc404991258602f0b7","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3b383ee2285332a194dbad5c697b0305b25dd8206e97711a9b282832753b74b4","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"d103024f5017fdcabaff08262102c5247078ca8f9b984031104c706e84e3556e","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547560068124672","occurred_at":"2026-09-25T18:09:53Z","signature_id":"0911bfb451cecb71d902c4d9a03521c3a5e579e6640a3eb871d2b06f1fb8defa","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103507194874957824","occurred_at":"2026-09-25T15:29:30Z","signature_id":"1890310e193288c8c01c75118137ccf125a485da275b0c15cc82d2f4e04e6b39","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","signature_id":"a9686068b550d21e25c4eca196ba3a0cfd9bb97605a83dcd0d2aca5dd9969175","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489841365782528","occurred_at":"2026-09-25T14:20:33Z","signature_id":"925b7298c77dbd0a35f322d27ca41279ec294430746a196070c77c282811bc46","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485306291359744","occurred_at":"2026-09-25T14:02:32Z","signature_id":"838df1a6da77eac70a576cc170cfe1289751436f04135f91c74f505e065a568e","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485064313573376","occurred_at":"2026-09-25T14:01:34Z","signature_id":"d41a7f88f22839705a091b2bf0ea143c79226ea00402747e93c227257fef7174","pr_number":7172,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":33,"runs_affected":33,"failed_runs":33,"run_impact_percent":4.667609618104668,"anchor_occurrences":33,"after_last_push_count":13,"trend_counts":[3,6,4,2,12,0,2,7],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":46,"prior_last_seen_at":"2026-09-27T13:27:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":33}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-29T00:14:18Z\n  Ended:       2026-09-29T00:26:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4504064-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T16:20:58Z\n  Ended:       2026-09-28T16:27:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0662784-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T12:00:53Z\n  Ended:       2026-09-28T12:14:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0514944-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T12:06:51Z\n  Ended:       2026-09-28T12:18:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0514944-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709623964504064","occurred_at":"2026-09-28T23:07:32Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7124,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529106451304448","occurred_at":"2026-09-28T11:10:13Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7182,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500902567088128","occurred_at":"2026-09-28T09:18:09Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104002487378776064","occurred_at":"2026-09-27T00:17:38Z","signature_id":"194da11ceda26cf004e70e30811c51ee51ab4977a0682ac50eca40b8c54129c3","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103507194874957824","occurred_at":"2026-09-25T15:29:30Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489696960090112","occurred_at":"2026-09-25T14:19:59Z","signature_id":"00ba874216a5343073a0adac3c644d98cfde892f61797b31ee3336d0f5f3c57e","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485377787465728","occurred_at":"2026-09-25T14:02:49Z","signature_id":"c162a5faff1d028c6c475696bbe945954abd7c58d8b8a99052f72cd3c91b0380","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103450358956167168","occurred_at":"2026-09-25T11:43:23Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103423928134275072","occurred_at":"2026-09-25T09:58:38Z","signature_id":"4126cd70f9e258007e4e457356ba9c9a093cd31c782620f814f33f5d61f037f4","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103419069259583488","occurred_at":"2026-09-25T09:39:20Z","signature_id":"d3dbde3657783287fac24beab9467fbbf74dc1c5cfa00e1645a47049ac447877","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318217182154752","occurred_at":"2026-09-25T02:58:35Z","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102965797063233536","occurred_at":"2026-09-24T03:38:11Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"125273ad41e8d32bfd551d1beb0bc45baf8569efc2fb05f6f2496e0c967d64ac","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"71f4c03d6bb4614602efd1761f8bbf73a703154b3346a44390285f109ca0b681","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424385783599104","occurred_at":"2026-09-22T15:46:49Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7079/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409098145304576","occurred_at":"2026-09-22T14:46:04Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7079,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","signature_id":"52816a71ef2a03b49578fdbb7a916c3c123b4226a4221c05983979cb58643e9c","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7060,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":31,"runs_affected":31,"failed_runs":31,"run_impact_percent":4.384724186704385,"anchor_occurrences":31,"after_last_push_count":10,"trend_counts":[0,0,0,0,6,5,8,12],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":31,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","occurrences":31}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:06:05Z\n  Ended:       2026-09-28T13:14:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:54:31Z\n  Ended:       2026-09-28T12:18:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:24:54Z\n  Ended:       2026-09-28T11:45:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"f41df77c38855e0775eecadb32a0db47fcca98f18d194ba1e30d8c87c6bfa835","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","signature_id":"d3bf0fab68d1955c534da4c546dfc7130a657d5faba2b998c4dfd5bfd50c00bf","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","signature_id":"68426bd9cacc1090d6633665a7e4523175f75051187e125f21864cda11e4b81d","pr_number":7218,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","signature_id":"adc03018ca13b9a200d0b335ac06a5da309ea6695cbde1885be48929840e5142","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104493553664135168","occurred_at":"2026-09-28T08:48:57Z","signature_id":"51b2e006e9652fd5900cb41a77a7d70291a4bee2a682adb677a4f1b150bc2599","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"4b99f55b3dddceb435e0a52e1d7443cc1d1476dd1512466b15df642e807fd310","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484510623076352","occurred_at":"2026-09-28T08:13:01Z","signature_id":"bb576c3e3075ea38f79306c173cad7ebcec01d4df44d1dd35aed1e3a8f78bd19","pr_number":7071,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","signature_id":"f0862ed51a1f4c7c2323f68ca122f5ce11e850a716c3b7c2fa2bb259aced13b0","pr_number":7214,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"ba7bab6629d6e8d805675d29c69fdec9a750497cd38dbfd826ca13c707567172","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"aa9f4214ab7044d7b9bcd462877e240cc060062645e93bcbaa17696de3b6d418","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104397716221071360","occurred_at":"2026-09-28T02:28:07Z","signature_id":"56897655307efa65f9f11fbfd619611b1b06b0460bb71bb172b6d5e7ce8f3d08","pr_number":7212,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","signature_id":"d98ef7ea866f42d46602defe307f0463377933bd3387492c8cad52c2448dec04","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"141dcb4a7bce83ddae2534a50b4037182ee14743f68296b8e0c644526062ca81","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","signature_id":"a4af37eae3845736c3bca8e48388195a8cf0fb39b9828c0799328f4f81338760","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","signature_id":"80970b397031adb31bef780349e99ab505e3f2fd73c9134ca7e696f1e795203c","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"426761ff9070cec28a5322afd36d3627552c3d22962de9e435913509b416aa8d","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"44a63252c6af03437dca2a2cd16633fffc635048800be390cd8ed0b750e906c9","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"35bed69d107d0e48eeae3149c2b40959f5c177d8606a49c69653469eb84a80bc","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"bdc02d01d20a4113becec172cb87ce69502ed7be7f54534e201fa4c1be3941cb","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"6493d354b73970d4e764ed97c72ddea6bbaa29dc9a60565e51aa0b12f093243d","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"6566df96cb73a9adffc87a980a6110cb9f6cb7fac4d1aef176d37f92cb479ed6","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"b687d77087ab0324f2e1d0aa267ff4136bba5841aa7af068e1fef75e22687772","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"e6f461584d644efe4659caaf9224f6ef200ce34ae3721cb24a9c26cca546b081","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881450792161280","occurred_at":"2026-09-26T16:16:40Z","signature_id":"028b8b3f149367175ed04cdc868d03ee9feeb1166c251ae199607b3d027a8634","pr_number":7188,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"29c97a1bc4096ae773b9213705339dee20d316902decd611365f2238cc2051e0","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","signature_id":"f80738046c4e039d573791df16b6cbb481153102a2bb23701d68c69a8a6a851d","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","signature_id":"6ddbf27f7eb384cd0a600dae6b3cf441fa8fbe9971573e883b0042de0375bd93","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103556819036147712","occurred_at":"2026-09-25T18:46:42Z","signature_id":"76c4a3536de55d2db8db1324950e8cbfeac28775f3a2d8d76e81d97a1126673d","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","signature_id":"e8f5b6bb718794f5ad75fa1fc52fd2b7f6cdb8275ea9e32cd99aac68f99b15e3","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","signature_id":"bfe979a56396aa9f080f5376ebac69b7f4a63495cb93eab7c4de33390c9d9d3f","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","signature_id":"14f1324b646a515c78bb9b0b3f9684f61e1dd4ba8de64e559e927d9fb3b94c61","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":29,"runs_affected":29,"failed_runs":29,"run_impact_percent":4.101838755304102,"anchor_occurrences":29,"after_last_push_count":8,"trend_counts":[0,0,0,0,5,4,8,12],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":29,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","occurrences":29}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:09:09Z\n  Ended:       2026-09-28T13:23:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:57:33Z\n  Ended:       2026-09-28T12:43:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:27:55Z\n  Ended:       2026-09-28T12:10:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n  Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"c42d12564755be56bc68f22fe0f502fb5d2224e2c44a2d94b8b4566c7d00cd5a","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","signature_id":"823850876940620727554ba4d2a603b64fa3d3d9b402a7fd7add96c0cc3b5926","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","signature_id":"52d0cc354a3b86402491a5b57b32bfdf6698b722ffc683b472b627660b31af7d","pr_number":7218,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","signature_id":"5db35e658cd311f66e3a715283d477fc14e8743c4f890d427402e8cb74dccf70","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104493553664135168","occurred_at":"2026-09-28T08:48:57Z","signature_id":"da5e7d6776d3b79243736efc00841e62ebccdd525b1a468588dd9309520d23c3","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"60ce8c4e66e5aca104e5530423691fb540d7e157b22d3d53163288d5c65dca6b","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484510623076352","occurred_at":"2026-09-28T08:13:01Z","signature_id":"2d8ac31341ad08baa5d71a8dcae5ad28f7794f7d50db6c4ebde370cafd3d436c","pr_number":7071,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","signature_id":"19f23baf6d87ff64419ebe3fa5185da1fce2564267135e1650412b08bae427b9","pr_number":7214,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"df04e28ceb24729f941b045e921b610aab82e4c5f836a237e9b786cc3ade9993","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"0a3cebc47b4d478e0e1704ea1b9313c7385a4dc3461b6d705f457131c1d86ad3","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","signature_id":"f85bd000c6ca2d2c6689999be19d81766a8baa6d766430077973ac505e4161a1","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","signature_id":"0746757cae55e1359ac94dc358bff8c1481c3785d7e5878a5a71e4eab63b083e","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"419f780bdbaa43f4ef8f4ecb0e33396b8bfad0e38a30159c1ff2da81cde2f1a7","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104315867142557696","occurred_at":"2026-09-27T21:02:53Z","signature_id":"d6d997e13699f3708b8da2df30da87216f574a5d894c86f3487f4f6d6d9ec091","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104291386130960384","occurred_at":"2026-09-27T19:25:36Z","signature_id":"ba0d797453f4acd2d3294ab03019f050a2ea64a58bfda6df78da0690b02a2c50","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"5afb798429b94f5c90e2fc1387a8558369362de6b6d0c8a51b724a0353effcc8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"8d21629c67d82189d892875373e45a37e10d571da501c0c8f7adab114af4262c","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"5e5cd04319eed373eaaf914f96f5fe47f4a0a57f723cd12e6afda7269f6982ed","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104201385095467008","occurred_at":"2026-09-27T13:27:59Z","signature_id":"6a3860a8f043002633cbc86fed6af04ae8f3209a874cc185b949e92160d28645","pr_number":7197,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"92318fcfac24b6233ea2adc5b233527abdcb8533537932bdc38c9c6b9dd86ffc","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7194/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103990456999219200","occurred_at":"2026-09-26T23:29:49Z","signature_id":"118b9a2c18bb76c55af60628010650bc62252eec1df7945fcd3c5d3154495df9","pr_number":7194,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"9ca4bc2deb7a2ca544935fb63e821804c0dc6f87fddd7c30fc0e84f823cb1c81","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881473420431360","occurred_at":"2026-09-26T16:16:46Z","signature_id":"7035fca2cdc8aba021f22051d3cd5b3870f8b1d6ff824a3d51afe42c3c74498a","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"8fb687948b9e439949059b6fd34880b83b1afcd34a8ef97d1f528299dcd7501c","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","signature_id":"c019ffe1f564224958ef7dd4995e2ce1348ec572e7f126e871a8a7be641c251e","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","signature_id":"c658821ff5f7d7b2d63a0e97b55950458ca17f792593dc632182ea487961cf11","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103556819036147712","occurred_at":"2026-09-25T18:46:42Z","signature_id":"a28ad0c15ed158b4ec6df29bb44cfd47d05ec01ee5dc565f62fa878852534253","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519763601821696","occurred_at":"2026-09-25T16:19:27Z","signature_id":"d4ed3df415fbe310f0efa530278cd9d07d7e76df5d066f5aaf8a375b9e6f0c9f","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","signature_id":"5ff26d2e2494532a8474d2c0bf3bb87491fd5743870688498d16fdf695c61721","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":29,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":29,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,0,29],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-17T20:37:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002154570\u003e: \n    resource group \"complex-cilium-kv-jndv4cm2shzf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jndv4cm2shzf\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\",\n      \"name\": \"e40876ab-da07-40cc-bca8-2d36902295a7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:31:44.581294657Z\",\n      \"endTime\": \"2026-09-28T19:55:45.796890355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rtrm9n8ddf2ubgd68c1s64e1ioap still exists (deletion dispatched at 2026-09-28T19:31:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002154570\u003e: \n    resource group \"complex-cilium-kv-jndv4cm2shzf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jndv4cm2shzf\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\",\n      \"name\": \"e40876ab-da07-40cc-bca8-2d36902295a7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:31:44.581294657Z\",\n      \"endTime\": \"2026-09-28T19:55:45.796890355Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rtrm9n8ddf2ubgd68c1s64e1ioap still exists (deletion dispatched at 2026-09-28T19:31:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013e4180\u003e: \n    resource group \"private-keyvault-bzhkhkm6vpw7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bzhkhkm6vpw7\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\",\n      \"name\": \"486f914e-d8e9-4103-988d-8630bbe515a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:29:30.639536976Z\",\n      \"endTime\": \"2026-09-28T19:53:35.789612068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rt8qnc76ptkosa2u3fv39hps3u1f still exists (deletion dispatched at 2026-09-28T19:29:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013e4180\u003e: \n    resource group \"private-keyvault-bzhkhkm6vpw7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bzhkhkm6vpw7\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\",\n      \"name\": \"486f914e-d8e9-4103-988d-8630bbe515a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:29:30.639536976Z\",\n      \"endTime\": \"2026-09-28T19:53:35.789612068Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rt8qnc76ptkosa2u3fv39hps3u1f still exists (deletion dispatched at 2026-09-28T19:29:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f1a240\u003e: \n    resource group \"rg-candidate-4-23-8lfxrmnvvv2w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8hk4r9\" in resourcegroup=\"rg-candidate-4-23-8lfxrmnvvv2w\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\",\n      \"name\": \"7972cdda-000f-4549-95ba-7b26044f70b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:49:57.362020562Z\",\n      \"endTime\": \"2026-09-28T20:14:00.292519621Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r6c80n9jpgk27jtqr1mg2ert1grv still exists (deletion dispatched at 2026-09-28T19:49:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f1a240\u003e: \n    resource group \"rg-candidate-4-23-8lfxrmnvvv2w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8hk4r9\" in resourcegroup=\"rg-candidate-4-23-8lfxrmnvvv2w\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\",\n      \"name\": \"7972cdda-000f-4549-95ba-7b26044f70b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:49:57.362020562Z\",\n      \"endTime\": \"2026-09-28T20:14:00.292519621Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r6c80n9jpgk27jtqr1mg2ert1grv still exists (deletion dispatched at 2026-09-28T19:49:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"6fe4cd9eafd43e0c6bf005c09b6f95529ba4135a257448638db3146d0a4fbe8a","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"781e63180597c1b1b938f67cc9e5dd0218e1b6f57f1eae6c6e300c0f327db76e","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"7b7a706cc0ff1c912898b8d7c4500fb2e9a9ce840a6f74f1d3c202e4b6252f67","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"fe3ed91dd61bff3602cc82c1188d31cd88e9091fc80ea4f192a5dbe2377f4f68","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"7f117a546143ac8ae4627871bef65f861c9cbed6350d628405581b5ef7dd1fa8","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"89f9db9301af68ce3054907127a6ba7b3d74e22731fd79fdc7a6996e89a63f9d","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"8f4ad0dcbe474318d38b8ceb4a2664e001ad4acd5b59bb3f9f027259298ec977","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"a6dfca451b27dfff658fc8f1a0157dcb703810b9eb1848556cf013c0e58694f8","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"b51f9ec76dc47f22d30731eb4fcdfa609aef5b3aaca327c9172c808dc60d7a3c","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"facbe030c0f80669a32838ce5a118445436a7e98d7047b622bb21b4ac7b42c09","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"05edd80a677f5d832c1ef05d940180d37fe21a3734cf35f46966ff517c506b0a","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"0fb3311ad2f9df382d87b904ce13084a4e580a3bdcc3cf253c032a2d12add8d1","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"228b55142d7421c5ea9202dfd88b3f5dbcf091d3cea1ff9bcb5da8cdf2018bba","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"2dfb5050c6aeff4f9fb910a04a4f8308c1afbe380de422b5d8c1203d2407f57e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"3575db45186608ce7e97cda0fd0cf74fa5dcc53098682cede51e13901add9b7f","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"48212f8bd40f9c5ee7d2e207dd0fc8d883c102f5d64b6cb526bcca292b3142d8","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"4d469d9ff8433380f5c21911bcde986dbc7542a7445ebb8ef34554faed2520d6","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"655632efe6eb665737f2fe79dbe081860714186ed63d5e7bbc7561be417fe6c4","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"67aee04b8869e7e7576385277fc0bb97f3b3eb5af8dd02fb45f04ec0a2550a5e","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"948424a2d7453ea605ce66e874ab718d017748efea90a195d896e62ef8689abb","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"d4bf4d27b2b15cf42ec27f335cd2d5cb9602474ad668a2b068ea83dd235f1bb1","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"e35a27569175c5bae78b2e4e738387abcfc6661dc3d77070c5b5ec75ed06600a","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"f86aa345da5041b62d0094051db665216f09f00381a536f35956b7dd0f86e650","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"168ed1ed23c3b84e54b2f0e49b128524a1e37906a697a2fe43f0b3969bdf66ae","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"8f9fa274992a586c59ef9b66ef85d395e00eb0397fa57b1ab28990ac84af7438","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"dca12c8dd7fe8182c1727d2cbd5b05ff33a4efdd1526d2cfc5c44292de098c00","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"2a44402a154f3d167466daa8cbf44d7db60e9fcc2662826bf741581bf798c662","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"927ea0a3feb6d2e147cb24c7843e4021b13aea7e7d34e9de65f81c89fd4479e6","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","signature_id":"968b6078567a646969495cf371728a4af6f1ffed8249b5773e68f74a12927a29","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":23,"runs_affected":23,"failed_runs":23,"run_impact_percent":3.253182461103253,"anchor_occurrences":23,"after_last_push_count":11,"trend_counts":[0,0,0,0,5,3,5,10],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-27T18:30:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","occurrences":23}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:06:06Z\n  Ended:       2026-09-28T13:14:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:54:32Z\n  Ended:       2026-09-28T12:18:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:41:58Z\n  Ended:       2026-09-28T11:49:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"0eaacea327a774d3a40fa905f3b0b3aa306082f3ce30942bab1dbcdad123919a","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","signature_id":"4782a74e9029e3cff62109ab7b8935f9b106a03d14ad0315f92c513a27406fce","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","signature_id":"ad4b6e564bf43ba35f8f2b2b54565b1dcd6add304f45b0a3ec90330d94c109cc","pr_number":7218,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104517513030144000","occurred_at":"2026-09-28T10:24:09Z","signature_id":"0402706d7b08da337e6c9a6ff7280d7bcc0c4516a43230d98d67a68ced2ef9a4","pr_number":7106,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"6789e672fb7d7a5c3e1c55f770176fb00fe56be9b227fabded289671c9ee231b","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104468013976653824","occurred_at":"2026-09-28T07:07:28Z","signature_id":"95dae5d8d35052e7be6830fe4168275bb671f71aab92c05a8181527ef6c63799","pr_number":7215,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"aef3898db187315c24458f77943698d78ec2e1a6b275d86eed3a6518338631b3","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","signature_id":"10f839c05a115a040846a911b08d87f6a5ef0e8b18e1acd188a5d746ed43ef16","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"9a3ac6241850f68acc835ada6fb9a30a428640bf55e1681d17c18117e623ace2","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","signature_id":"0e04fa5714858ca45f181fa5b8f6c4124ef2770063cbac0e9be7b8c170fcd30e","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"893ce1f9fd178961fa552c9c4ed69700c8bd2c12d9b189640fba68210e5e2004","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"a3273e0d1794b7c8a67f767205c53f73d87ec1d167083395847a4aaed230b76f","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"9d4a34fd40b7a4768d4e4eabe4a4ec42749a225062d6842b119ed23e64fabb54","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"0bafa741a68390f4b6bf980e8d38b5adb37e713e43141fb95567a5f642ddcfdd","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"c79fa012a0b071c6064bdd63844fb3a758f0f6a2bc109b1ebc88b8a973287f91","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"3a14be93314f64bd1aaf34287e4e0adef28fca1a41e77dc2ba5a7194956ee79f","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"11ba818ac5dd1ff347068d274c7fa5d21dba08c20baa52b6b0f46bbac6c58411","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5afa5c76e03a3b0c3b55adc55370d70f4ab3538b83cc562c8d049fb3ad18fe8a","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","signature_id":"59c796fbf53347a929d1140f136d97e8f215af00e5e2f39835a61c93eeea0c68","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","signature_id":"610d97b5c38aad5358d95f8441cb315dd1aae476119a3ff7e9090737558bd925","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","signature_id":"17ad0b9bb6164e3d6001f2d6865aa89ffb225847638c9542425a50a55f53006d","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","signature_id":"111b6f7f301fb2f1efa95a5ddf7cc001a54bed1e2f2b096078dd112e0cbf027b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","signature_id":"2570a67933cbb17e5fa7c5faafbae18b8522372cb31dbe440c9e1e0a8bdc7c05","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":22,"runs_affected":22,"failed_runs":22,"run_impact_percent":3.1117397454031117,"anchor_occurrences":22,"after_last_push_count":10,"trend_counts":[0,0,0,0,4,3,5,10],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":22,"prior_last_seen_at":"2026-09-27T18:30:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","occurrences":22}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:09:10Z\n  Ended:       2026-09-28T13:39:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:57:33Z\n  Ended:       2026-09-28T12:43:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T11:44:58Z\n  Ended:       2026-09-28T12:14:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n  Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"b254aaa1635ed84a93daf87b215afa6a9984a990ff39d417d58e92d00c86be46","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","signature_id":"7630ab66e6fcc7b1b99335513790b90b490d52ddfbc4f6082579503d0e184b53","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","signature_id":"a3308501a5c9a7128ceab2567a69a869e6a77d0446039c5c3aacec9cc2b85dba","pr_number":7218,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104517513030144000","occurred_at":"2026-09-28T10:24:09Z","signature_id":"3c33a5425afe4d1a10fafe1ea63cdaa957078a492eb938968ad4b5da432c09d0","pr_number":7106,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"a7a4c6a0026c3f599f6b4f74fe252b7b2b73f3f52e4f6fcefc52ad3c0977d311","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104468013976653824","occurred_at":"2026-09-28T07:07:28Z","signature_id":"b3655bb5b695bdc2e0832b9062b0fa6d3aa5dfa0ae58d7d00560b741e3fab4a2","pr_number":7215,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"84600d415d69723b878c3431a0471785536256c27ac395972895dceb4cac7a86","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","signature_id":"6b2918236ea1d95b7e6826a1ce35aa62303eaac8605953f4e310050355cbd27e","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"5e3d7814d9c9dfb8b95ab505c063df649c87d6d6942b6122da0574d5b0d3386c","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","signature_id":"d1fe2c91f414126eebe9264ae5312e44cb5f74a34c9a0915d3a6a984b56ea6aa","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104277605581590528","occurred_at":"2026-09-27T18:30:51Z","signature_id":"191a6f3923b9adfd1af697a8104aad4e301e37d24962d749afe729bbd97e8f3a","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"66132c0ef28f02375a925f830a63f89a139e8421a3d4bf3c8d7b497dcbd25498","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104109345670172672","occurred_at":"2026-09-27T07:22:15Z","signature_id":"0ae2d2740aeab65c0927cc3b7b9751bc45171ae3ffe4aada6c1848ef04948ef5","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071277122686976","occurred_at":"2026-09-27T04:50:58Z","signature_id":"c1605d1431fa38cffcc889bc3b160bbc5879105dd589db8bc2b28a285e29f8b6","pr_number":7174,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"33e8f05c23ee905d13012921f05b90827ce9b99ceb5852cf28e896b69118e778","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"878ce001cd49a7ed748478aef200a126d7f5d5a033ae0db1aa1de44d5ddcc877","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"1d021545a8594032e7b33f8a228af4da144b809174ff6174498d77fa92101d7c","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"983ca4f9ae3ed26808cbee2ea2a4466c45d7f3a39a19058178e4a59512e41739","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","signature_id":"e3101a2b58ad0aba9cc14b18b48bd4af2812352abeb2151468084aebc395f66f","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103526925438816256","occurred_at":"2026-09-25T16:47:55Z","signature_id":"74115cd9a87272e438969badac47448ecb69890d765eba6086cb2e71bf1e9e4e","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","signature_id":"301e2cf0d90497c804a3b815b89a2f5d8dbae118206c74ca0baaf6b49f6f0d9b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519651102199808","occurred_at":"2026-09-25T16:18:52Z","signature_id":"009f41ec16856f6a2f163a6d76e1833343da4f507cba32812733d4138afdbf21","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":22,"runs_affected":22,"failed_runs":22,"run_impact_percent":3.1117397454031117,"anchor_occurrences":22,"after_last_push_count":20,"trend_counts":[4,13,3,5,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":37,"prior_last_seen_at":"2026-09-25T08:41:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":22}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T10:21:49Z\n  Ended:       2026-09-25T10:27:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6461184-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j6461184-svc).\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-25T10:32:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j6461184-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/providers/microsoft.redhatopenshift/locations/{location}/hcpoperationstatuses/{operationid}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/providers/microsoft.redhatopenshift/locations/{location}/hcpoperationstatuses/{operationid} exceeded 1 second over the past 30 minutes (cluster ci01-j6461184-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T17:31:32Z\n  Ended:       2026-09-24T17:39:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1914624-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j1914624-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T12:40:11Z\n  Ended:       2026-09-24T12:46:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4440832-svc\", component=\"frontend\", method=\"post\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4440832-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103404569546461184","occurred_at":"2026-09-25T08:41:42Z","signature_id":"50d133ff46da6182299e88907232ea570e9476c64cd6c905b70efc73336787e6","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"0e8f57e37d73e9d51e134e1e0ceb298fcf33a07521e65f7fbe44784034a97653","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"52a5a38b47cc6e69825210c368c8f32d8cf5f29e6283dbea51644a315c099800","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015428132442112","occurred_at":"2026-09-24T06:55:24Z","signature_id":"609ef47bc200be1ff06713d78b2540ac62b150a8d0e17b016ed8c4fd89fa13fe","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"e02ac96b476af77a18c8814f0b86746bd97b987ea683b9a1aa418df78ed20adc","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"7715e65904649de36554c3fa389bc722801fea292dc922d83d7ddf72b37313e4","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"d5d990830365516d1467804cae51c493981b29d7c2296cdf7adf893366ce4712","pr_number":7110,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","occurred_at":"2026-09-23T17:28:39Z","signature_id":"c1db43e1f9ed556a1e717aa9ea521e513132026799cb40e29c7cc1976942dff0","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","signature_id":"b20cadb846b55234efc43634cdf321fda9be13c6186765f74dd9bcf81c873e27","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534076756922368","occurred_at":"2026-09-22T23:02:41Z","signature_id":"3ed4d9b039f61b590fffd2d094c8a21584d0052cbfd2d7647bd6c94de416a2ca","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7089/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530666527723520","occurred_at":"2026-09-22T22:49:08Z","signature_id":"e63f01b8163ebef955b0612083617e40ee244e5eac4deff580529f1300de341a","pr_number":7089,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7084/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102495173459054592","occurred_at":"2026-09-22T20:20:22Z","signature_id":"214335afe92282c046d1950ce30a5f52d2bcbdf9d09adec832e75fe415c69131","pr_number":7084,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102443148557946880","occurred_at":"2026-09-22T17:01:22Z","signature_id":"fcc48db5d3cd87f0fff18e34375d136c7b3ab27aa3495f73b687e0b8a00d91a7","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","signature_id":"7ec1bacfd5828bea1dddf53be5010f15e6e95874c8514734962d8e8dfced23fe","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","signature_id":"a76dfeabb77ec9c40d970f39e98e5c3d30889b460b25840fc635fa5a5c93c4b9","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"2b6f0a14ed8cae2c4deeed2fde914673ffa2aeea1f351712769b94ee40fff620","pr_number":7066,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322606454607872","occurred_at":"2026-09-22T09:02:23Z","signature_id":"ad66a2666c34b923be42f086c71ddfb6acbad8f1b4774fa7a1f536a33d881b25","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","signature_id":"d87de921cb349c4db01e354fe18d2469501ad7fab0e50dc1fe219f4ec2f384c1","pr_number":6516,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102295446469742592","occurred_at":"2026-09-22T07:14:27Z","signature_id":"e0e17143e5d3902f1f452df7ad652fc0f85ccf61ea0d86668034f00a58d0fea8","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","signature_id":"a1415df35e18dcf241dab67e086a78b1cecb3327d794d9305875cc8e410a17fb","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231550941728768","occurred_at":"2026-09-22T03:00:33Z","signature_id":"454a09f778a2169168b336e5d77f9b9de90a7344d98888a2101b0f7ede0e3c84","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186809273356288","occurred_at":"2026-09-22T00:02:45Z","signature_id":"8c975fd9f886200ee26bc3444b0583309d21a1e572b61a74aaf57cf9297a81e7","pr_number":7053,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":2.545968882602546,"anchor_occurrences":18,"after_last_push_count":51,"trend_counts":[2,2,2,4,6,1,2,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":104,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":18}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T06:51:50Z\n  Ended:       2026-09-28T06:57:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1154560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3rl3f8tj7aka16mnasqurrb5ebbvn0-w7g1w7z1p7g5a8x\", pod=\"kube-apiserver-8448d69f6f-jqnnb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3rl3f8tj7aka16mnasqurrb5ebbvn0-w7g1w7z1p7g5a8x/kube-apiserver-8448d69f6f-jqnnb has been in a non-ready state for longer than 5 minutes.","alert fired 13 time(s) (6 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:06:28Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:06:29Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:07:30Z\n  Ended:       2026-09-27T22:48:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:09:27Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:10:30Z\n  Ended:       2026-09-27T22:49:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T17:37:18Z\n  Ended:       2026-09-27T17:43:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1132032-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j\", pod=\"router-5584fc994b-5x9hp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t3g642vb3ovmhl4qiumvarv9qifaek4-c6e1u6h4n5k3a7j/router-5584fc994b-5x9hp has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","signature_id":"f7140b732d2b9ada8666aae03ac943606c09447431321d689e9e3f694f2a46d7","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"7c6c23658687695d208580c0620f85e276d1932e76e62b017338962064930246","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104246785001132032","occurred_at":"2026-09-27T16:28:23Z","signature_id":"bd2bb1667dcc46addf88c81f320c44761ca728d893e15805a3bd4ad0673a41d2","pr_number":7199,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881428969197568","occurred_at":"2026-09-26T16:16:35Z","signature_id":"4887464f22fd02cf878c19ad54c98e5ffd7550a5bce16e494d595bda58002c06","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"1e74d40e28446d51c4e5d1116f6b0a7f045854ce87bcd798968ad60cec419685","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461761565855744","occurred_at":"2026-09-25T12:28:58Z","signature_id":"16462dfb1b7ae544495c5b7da4413d3ef0ee0bc77c796be4c8df88ca7c8dbf0a","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"0a0c47e7fa898ae28c5add2df96849779c38742a25e77d6ee2c158a5dcead973","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","signature_id":"ddf8ccdb3122a4b47c1d6e20ff22259362cf0d7b2c3a63d9a23d1d7a7a27f9c9","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","signature_id":"971c1ccdcbd740d38dcc070a46e71996dd9726524b859ca87241878c2670d18c","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","signature_id":"4b57206254fb7de2fa5b705bd0848fd62b6ad1452a8b7a4204cadac985929aef","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"9148700f82fd5b370a512210ca6a6cb677852f9f2d8abcc81bf3649ab5047f43","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"7fbc78ba550f54a61f9bb8680a644c7d2c72d2a80281632662dd5cddec3c05d7","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7130/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103111106493943808","occurred_at":"2026-09-24T13:15:33Z","signature_id":"71628d4204794d2bf5a1e409f017e03485e53450d5f8ef544bb7d4efc8d8ed17","pr_number":7130,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","signature_id":"eb34a8b64fb9114676f2191bea2c8d39236f5170336f8b7d13fb0795cc5edc83","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"9766aea3f661a15c4b1ef2c8ce948e7447e6a77d814fe12d510f6ddd5782ae00","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","occurred_at":"2026-09-23T05:02:00Z","signature_id":"fec44601492cd080815bfd1f2fda9d583e0a942b12d71ab0789e80aa75b0494f","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"833f40b32f2367c7a516e338150ce8ef38e4e99136c3b8d58fb87da8e3aac23d","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","signature_id":"2f5c275c19af39d433c7a0ffe224f7e5c2c3268c6bbe4ea0fae334b7478f8fe6","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":2.545968882602546,"anchor_occurrences":18,"after_last_push_count":5,"trend_counts":[1,3,2,4,1,2,1,5],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":26,"prior_last_seen_at":"2026-09-27T13:08:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":18}],"full_error_samples":["alert fired 79 time(s) (63 unknown, 16 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T19:22:59Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"router-5b97c8c479-td767\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/router-5b97c8c479-td767 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T19:23:58Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rbkaemrkauofk92ubess3a0bmc8a-v20261001\", pod=\"router-688fdf9d74-7b7qb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46rbkaemrkauofk92ubess3a0bmc8a-v20261001/router-688fdf9d74-7b7qb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-28T19:25:04Z\n  Ended:       2026-09-28T19:47:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster\", pod=\"router-7856497d5-p9s72\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster/router-7856497d5-p9s72 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-28T19:25:04Z\n  Ended:       2026-09-28T19:47:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster\", pod=\"router-7856497d5-ptl86\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster/router-7856497d5-ptl86 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"ovnkube-control-plane-5fd96b4979-gbmgt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/ovnkube-control-plane-5fd96b4979-gbmgt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:46:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"ingress-operator-7d88f7b5dc-drxfd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/ingress-operator-7d88f7b5dc-drxfd has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:33:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-file-csi-driver-controller-854b478df8-v98lf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-file-csi-driver-controller-854b478df8-v98lf has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:32:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-disk-csi-driver-controller-5785d6645d-6xw4f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-disk-csi-driver-controller-5785d6645d-6xw4f has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:32:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-disk-csi-driver-controller-5c84fd46c4-mpzts\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-disk-csi-driver-controller-5c84fd46c4-mpzts has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:33:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-file-csi-driver-controller-559d66d9f8-v468g\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-file-csi-driver-controller-559d66d9f8-v468g has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:33:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-file-csi-driver-controller-9688c9fcd-vkkwc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-file-csi-driver-controller-9688c9fcd-vkkwc has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:32:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-disk-csi-driver-controller-df8985bcf-xr5pq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-disk-csi-driver-controller-df8985bcf-xr5pq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-28T19:25:59Z\n  Ended:       2026-09-28T19:32:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-file-csi-driver-controller-6f4b9c57cd-gpkcq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-file-csi-driver-controller-6f4b9c57cd-gpkcq has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-28T19:29:02Z\n  Ended:       2026-09-28T19:35:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6qhsdqg59jb18mtn75mdepd994t-p1e0l2c6g7e7x9p\", pod=\"azure-disk-csi-driver-controller-d5b4b46ff-swrjk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6qhsdqg59jb18mtn75mdepd994t-p1e0l2c6g7e7x9p/azure-disk-csi-driver-controller-d5b4b46ff-swrjk has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-28T19:29:02Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"ignition-server-6cf4c4cdbb-dcl2n\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/ignition-server-6cf4c4cdbb-dcl2n has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-28T19:30:02Z\n  Ended:       2026-09-28T19:36:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"cluster-storage-operator-665d6949b6-6fw7x\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/cluster-storage-operator-665d6949b6-6fw7x has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-28T19:30:02Z\n  Ended:       2026-09-28T19:37:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"ingress-operator-fb65c85-29jmc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/ingress-operator-fb65c85-29jmc has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-28T19:30:02Z\n  Ended:       2026-09-28T19:36:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"dns-operator-bfc958595-f2lvq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/dns-operator-bfc958595-f2lvq has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-28T19:30:02Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"kube-controller-manager-847bbfc79-gdtt6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/kube-controller-manager-847bbfc79-gdtt6 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-28T19:30:02Z\n  Ended:       2026-09-28T19:36:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"cluster-network-operator-6995f687db-6cfj4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/cluster-network-operator-6995f687db-6cfj4 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-28T19:33:01Z\n  Ended:       2026-09-28T19:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rb6d2v8ktanhkbba2l3aora65d7u-t1b3k0j9k6s4g3e\", pod=\"kube-apiserver-5f69cc7c69-66mrs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46rb6d2v8ktanhkbba2l3aora65d7u-t1b3k0j9k6s4g3e/kube-apiserver-5f69cc7c69-66mrs has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-network-operator-79b745b7-2lm4f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-network-operator-79b745b7-2lm4f has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:46:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"kube-apiserver-c7844b576-98nm5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/kube-apiserver-c7844b576-98nm5 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-version-operator-767f75f6bd-czn9v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-version-operator-767f75f6bd-czn9v has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"network-node-identity-688597d98d-5nwfj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/network-node-identity-688597d98d-5nwfj has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:46:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"kube-apiserver-674ffb8fcf-w2vnq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/kube-apiserver-674ffb8fcf-w2vnq has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"csi-snapshot-controller-78d9d968f6-lktl7\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/csi-snapshot-controller-78d9d968f6-lktl7 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"kube-apiserver-7f4977d86c-ggtv4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/kube-apiserver-7f4977d86c-ggtv4 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:44:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"community-operators-catalog-77957f8659-vrvjj\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/community-operators-catalog-77957f8659-vrvjj has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:44:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"multus-admission-controller-6659c5d88f-hgsf6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/multus-admission-controller-6659c5d88f-hgsf6 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"router-7964b968d8-s9n7t\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/router-7964b968d8-s9n7t has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"ignition-server-6cf4c4cdbb-xqn6v\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/ignition-server-6cf4c4cdbb-xqn6v has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"openshift-apiserver-79bff85c55-rmr6j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/openshift-apiserver-79bff85c55-rmr6j has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:44:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"openshift-apiserver-f4f5cc6bf-j27d5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/openshift-apiserver-f4f5cc6bf-j27d5 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:44:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r619d7598ajep6f8mtkq1cmmun0r-g6f2d0d9e8y9l8w\", pod=\"openshift-apiserver-855bf867d4-5dj69\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r619d7598ajep6f8mtkq1cmmun0r-g6f2d0d9e8y9l8w/openshift-apiserver-855bf867d4-5dj69 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Resolved\n  Started:     2026-09-28T19:34:01Z\n  Ended:       2026-09-28T19:45:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"openshift-apiserver-548d95f77-j9rt4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/openshift-apiserver-548d95f77-j9rt4 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:46:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"kube-apiserver-8488cbccb8-9cntq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/kube-apiserver-8488cbccb8-9cntq has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:44:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"openshift-apiserver-59cf74f469-2pkbk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/openshift-apiserver-59cf74f469-2pkbk has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:44:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-image-registry-operator-5f9967bf67-m75d8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-image-registry-operator-5f9967bf67-m75d8 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"router-6966cb469f-jdx2g\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/router-6966cb469f-jdx2g has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:46:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"kube-apiserver-6474ccdcd7-j5pkn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/kube-apiserver-6474ccdcd7-j5pkn has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"kube-apiserver-7877d486f4-dwd29\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/kube-apiserver-7877d486f4-dwd29 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"network-node-identity-79f47f8db7-c6lsb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/network-node-identity-79f47f8db7-c6lsb has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:45:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"network-node-identity-6bf46b86d-r6lxm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/network-node-identity-6bf46b86d-r6lxm has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:48:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"router-d5fd8c684-npjp8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/router-d5fd8c684-npjp8 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"openshift-apiserver-7fccc5c996-2fqjh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/openshift-apiserver-7fccc5c996-2fqjh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n  State:       Resolved\n  Started:     2026-09-28T19:35:03Z\n  Ended:       2026-09-28T19:44:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"certified-operators-catalog-56fcb687d5-x55km\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/certified-operators-catalog-56fcb687d5-x55km has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"openshift-apiserver-58689dc575-d9psz\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/openshift-apiserver-58689dc575-d9psz has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"openshift-apiserver-fb96d7cd5-n8vx8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/openshift-apiserver-fb96d7cd5-n8vx8 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"kube-apiserver-58866c8f4c-rssbm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/kube-apiserver-58866c8f4c-rssbm has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"openshift-apiserver-57597ccd79-kdsdf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/openshift-apiserver-57597ccd79-kdsdf has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"network-node-identity-6b45dfb8b5-gpsw6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/network-node-identity-6b45dfb8b5-gpsw6 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"router-5b97c8c479-87d9j\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/router-5b97c8c479-87d9j has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"network-node-identity-cf9fb8b88-4lqsp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/network-node-identity-cf9fb8b88-4lqsp has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n  State:       Resolved\n  Started:     2026-09-28T19:35:04Z\n  Ended:       2026-09-28T19:45:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"kube-apiserver-6776ddb8b4-jtcfw\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/kube-apiserver-6776ddb8b4-jtcfw has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n  State:       Resolved\n  Started:     2026-09-28T19:36:01Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-file-csi-driver-controller-68d9cdbd6c-bjqhs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-file-csi-driver-controller-68d9cdbd6c-bjqhs has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n  State:       Resolved\n  Started:     2026-09-28T19:36:02Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-disk-csi-driver-controller-688579f7b8-gcwdq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-disk-csi-driver-controller-688579f7b8-gcwdq has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n  State:       Resolved\n  Started:     2026-09-28T19:36:02Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-disk-csi-driver-controller-778588d6c6-grn9w\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-disk-csi-driver-controller-778588d6c6-grn9w has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n  State:       Resolved\n  Started:     2026-09-28T19:36:02Z\n  Ended:       2026-09-28T19:45:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"kube-controller-manager-5488d69fb6-p9ltk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/kube-controller-manager-5488d69fb6-p9ltk has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n  State:       Resolved\n  Started:     2026-09-28T19:37:01Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"kube-controller-manager-9864948dd-qzgnt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/kube-controller-manager-9864948dd-qzgnt has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n  State:       Resolved\n  Started:     2026-09-28T19:37:01Z\n  Ended:       2026-09-28T19:44:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"ovnkube-control-plane-58c8f9b6c5-45j6m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/ovnkube-control-plane-58c8f9b6c5-45j6m has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n  State:       Resolved\n  Started:     2026-09-28T19:37:01Z\n  Ended:       2026-09-28T19:44:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"network-node-identity-789b89cf4b-2v5sk\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/network-node-identity-789b89cf4b-2v5sk has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n  State:       Resolved\n  Started:     2026-09-28T19:38:02Z\n  Ended:       2026-09-28T19:44:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re3hoaml91vd90m68oiaekbg44as-e7l1h2p9d0o6f4j\", pod=\"openshift-apiserver-5f8cc55fd4-bm995\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t46re3hoaml91vd90m68oiaekbg44as-e7l1h2p9d0o6f4j/openshift-apiserver-5f8cc55fd4-bm995 has been in a non-ready state for longer than 5 minutes.","alert fired 39 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qwlf6\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-qwlf6 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-w28wh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-w28wh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-zxx8r\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-zxx8r has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-g76qj\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-g76qj has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-d9qzz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-d9qzz has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-fpkv2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-fpkv2 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-cwzx8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-cwzx8 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qm6c8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-qm6c8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-4996z\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-4996z has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-675f5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-675f5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-gd9k9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-gd9k9 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-z58gp\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-z58gp has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-4j2hv\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-4j2hv has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-8n4fj\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-8n4fj has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-8q8gs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-8q8gs has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-ddxpz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-ddxpz has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-9kqwf\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-9kqwf has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-tpk28\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-tpk28 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-wpwxf\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-wpwxf has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-c9pkv\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-c9pkv has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-r2z8h\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-r2z8h has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-v6lst\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-v6lst has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-6nc22\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-6nc22 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-52zbx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-52zbx has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-j8nbh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-j8nbh has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-f9lhx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-f9lhx has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dp559\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-dp559 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-j4pwc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-j4pwc has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-p9w7c\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-p9w7c has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-hqdp9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-hqdp9 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dgb65\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-dgb65 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-rvk8h\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-rvk8h has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-mpfhk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-mpfhk has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qm2ts\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-qm2ts has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-27cpw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-27cpw has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-mxsvz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-mxsvz has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-hgtjs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-hgtjs has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-c8pzf\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-c8pzf has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T16:03:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dvzkd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod monitoring/kube-events-7bcbfb996f-dvzkd has been in a non-ready state for longer than 5 minutes.","alert fired 54 time(s) (38 unknown, 16 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:06:01Z\n  Ended:       2026-09-28T13:29:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41bfp4l0n502nn624hggcdkif3fakm-l7r5c9r0t4i6i9i\", pod=\"router-5f8dd5cdc8-2x7hv\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41bfp4l0n502nn624hggcdkif3fakm-l7r5c9r0t4i6i9i/router-5f8dd5cdc8-2x7hv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T13:07:01Z\n  Ended:       2026-09-28T13:13:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"cluster-network-operator-745b94c8d6-xgbrm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/cluster-network-operator-745b94c8d6-xgbrm has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-28T13:07:01Z\n  Ended:       2026-09-28T13:29:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster\", pod=\"router-7f5958d584-h7qql\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster/router-7f5958d584-h7qql has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-28T13:07:01Z\n  Ended:       2026-09-28T13:26:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"openshift-apiserver-77dbbb84d4-g84qr\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/openshift-apiserver-77dbbb84d4-g84qr has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-28T13:07:01Z\n  Ended:       2026-09-28T13:32:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41aorgo7k4e5t9d3gl5386u8qrr1ok-b1g2t3h0x4w9w8x\", pod=\"router-869d798bbd-2c6kt\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41aorgo7k4e5t9d3gl5386u8qrr1ok-b1g2t3h0x4w9w8x/router-869d798bbd-2c6kt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-28T13:07:01Z\n  Ended:       2026-09-28T13:37:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster\", pod=\"router-7f5958d584-xsp4j\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster/router-7f5958d584-xsp4j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-28T13:08:02Z\n  Ended:       2026-09-28T13:35:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41bju2fkvns9a7dk4sqfvummalvm0f-t0e8p3i4e9p7l9u\", pod=\"router-6449799f6-vwm42\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41bju2fkvns9a7dk4sqfvummalvm0f-t0e8p3i4e9p7l9u/router-6449799f6-vwm42 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-28T13:08:02Z\n  Ended:       2026-09-28T13:26:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"ignition-server-proxy-67ffb7567d-ndltn\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/ignition-server-proxy-67ffb7567d-ndltn has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-28T13:08:03Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41an05susenrnl0dfs40rcas06ch4c-f7z5i4t9g6o0n5p\", pod=\"ignition-server-proxy-8699545598-fvwm5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41an05susenrnl0dfs40rcas06ch4c-f7z5i4t9g6o0n5p/ignition-server-proxy-8699545598-fvwm5 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-28T13:08:03Z\n  Ended:       2026-09-28T13:26:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-controller-manager-867d6bcbc8-7zqfz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-controller-manager-867d6bcbc8-7zqfz has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-28T13:16:02Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"openshift-apiserver-db5f58d87-n2wk7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/openshift-apiserver-db5f58d87-n2wk7 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"kube-apiserver-85d5846f96-gpf28\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/kube-apiserver-85d5846f96-gpf28 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", pod=\"openshift-apiserver-85677f65c4-wrvhg\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics/openshift-apiserver-85677f65c4-wrvhg has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"network-node-identity-7549b9758b-4wb6s\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/network-node-identity-7549b9758b-4wb6s has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"openshift-apiserver-77dbbb84d4-29r9g\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/openshift-apiserver-77dbbb84d4-29r9g has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-apiserver-5c98b4447b-wvxv8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-apiserver-5c98b4447b-wvxv8 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"kube-apiserver-745ff9b4db-srgc7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/kube-apiserver-745ff9b4db-srgc7 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"openshift-apiserver-84fcf4bc4d-mcmsx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/openshift-apiserver-84fcf4bc4d-mcmsx has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"openshift-apiserver-b6b777476-w4j8h\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/openshift-apiserver-b6b777476-w4j8h has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"network-node-identity-57484c8884-trf2c\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/network-node-identity-57484c8884-trf2c has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"openshift-apiserver-5c68654d4b-dbjjt\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/openshift-apiserver-5c68654d4b-dbjjt has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"openshift-apiserver-68c54f7686-xjxd2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/openshift-apiserver-68c54f7686-xjxd2 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"kube-apiserver-86c59c88cb-qrxm8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/kube-apiserver-86c59c88cb-qrxm8 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-controller-manager-867d6bcbc8-bpsb5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-controller-manager-867d6bcbc8-bpsb5 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"network-node-identity-65ff54759c-92pcp\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/network-node-identity-65ff54759c-92pcp has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"network-node-identity-594c4dccf4-lm2n6\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/network-node-identity-594c4dccf4-lm2n6 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", pod=\"kube-apiserver-b8d4f754f-dqwb6\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics/kube-apiserver-b8d4f754f-dqwb6 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"kube-apiserver-957c77ffc-rg8wb\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/kube-apiserver-957c77ffc-rg8wb has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"kube-apiserver-5c9ff4f96c-n5vjw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/kube-apiserver-5c9ff4f96c-n5vjw has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"openshift-apiserver-554784946b-mjxzz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/openshift-apiserver-554784946b-mjxzz has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Resolved\n  Started:     2026-09-28T13:17:07Z\n  Ended:       2026-09-28T13:26:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"kube-apiserver-65f96cf6cc-d9g85\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/kube-apiserver-65f96cf6cc-d9g85 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Resolved\n  Started:     2026-09-28T13:19:08Z\n  Ended:       2026-09-28T13:26:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anli60mk7obvb9qscekiln1r7nlh-z0b5k9r4h8u4l9q\", pod=\"openshift-apiserver-5f4c5b7f7c-l5gpc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41anli60mk7obvb9qscekiln1r7nlh-z0b5k9r4h8u4l9q/openshift-apiserver-5f4c5b7f7c-l5gpc has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Resolved\n  Started:     2026-09-28T13:20:08Z\n  Ended:       2026-09-28T13:26:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"multus-admission-controller-78665bcc7c-5f2bh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/multus-admission-controller-78665bcc7c-5f2bh has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Resolved\n  Started:     2026-09-28T13:31:01Z\n  Ended:       2026-09-28T13:41:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"csi-snapshot-controller-operator-74759598d-8dhlz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/csi-snapshot-controller-operator-74759598d-8dhlz has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Resolved\n  Started:     2026-09-28T13:31:01Z\n  Ended:       2026-09-28T13:41:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"cluster-storage-operator-55cc4d6675-r7fxd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/cluster-storage-operator-55cc4d6675-r7fxd has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Resolved\n  Started:     2026-09-28T13:31:01Z\n  Ended:       2026-09-28T13:41:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"cluster-network-operator-5dc56d94dc-rmrjd\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/cluster-network-operator-5dc56d94dc-rmrjd has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Resolved\n  Started:     2026-09-28T13:31:01Z\n  Ended:       2026-09-28T13:41:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"dns-operator-6d45bd4dc-c5h9t\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/dns-operator-6d45bd4dc-c5h9t has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Resolved\n  Started:     2026-09-28T13:31:01Z\n  Ended:       2026-09-28T13:41:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"ingress-operator-6c4557f7b9-x9crg\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/ingress-operator-6c4557f7b9-x9crg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"628ec6c53961c8770e007de15e33a4677fa2c411d6198e11b35c424d814ac61c","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","signature_id":"a0f3783cbc9752f6f287380baf21644ed565dc45c0b9da951cea46157a26275a","pr_number":7143,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"0c6477cbe2fa9f416a53e241e5f45eeea8fd5de09d670a995c600c8dd10a36a0","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"11ffe7254f8c26fa108e93294ba03af128f400124607c21f497d84718b3c6125","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"bb574e36c9b0b8ab6f482dba45ea40cb2e48e37b1dc20817e4c37c42065ee2f6","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104196521745453056","occurred_at":"2026-09-27T13:08:39Z","signature_id":"325bf50ff4030e0a9e49b5267b69a16cbb66d95e03425b035183cf4895a04dc3","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"f6e1a0738cb75e810c1d760e4593e47ecd222a26626a4f413d3bafc5f44c9529","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929857506283520","occurred_at":"2026-09-26T19:29:01Z","signature_id":"a5c5601829d7dc2e22894167e287975c02c8a19694717b357e8b26e5dffcb3d2","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"23ecffc8f4b7c15d8ff10d4272fefe60a14a999143507f8a644a0af9081b9865","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"39ce01c9a16cf6ef63990abb5b52786f61c0e8bb85c12b3c8bff45bc21ee3246","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"49174f0842b090006dcda4660be00bccd297316410781deb14da7700d135010d","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8a9c66eda94e766b0ef899f88f4ac1a07026419074e6b27f42b8d0218d54c88e","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921738877669376","occurred_at":"2026-09-24T00:43:07Z","signature_id":"3b853c2ad740c6b27854bb7b28c66560f90ec20a918beecff56ae64e308ded54","pr_number":7090,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"270a9593b205f1bd6398600a798ec9e0946dfbdebb1375f7feaf55d764c6e286","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"cd2acdca3a5c4271f034e124fe4ffb19ca10970d45a2e64d9db71f66d0c505d1","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","signature_id":"e5c46939c2f0bd4411d5eb2f2df37dbc82ead1a90ce003059981d14a1a7c466f","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","signature_id":"787c181d5ab0cf85f109f8fa52f4249050537f88d2a9185e8486e591858ba600","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","signature_id":"9e6b361c954476898c5522ff330dd04d38e832d28242edb73b886c538ec0826c","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":18,"runs_affected":18,"failed_runs":18,"run_impact_percent":2.545968882602546,"anchor_occurrences":18,"after_last_push_count":9,"also_in":["PROD","STG"],"trend_counts":[0,2,0,1,0,4,4,7],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":28,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":18}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6922,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104492976943140864","occurred_at":"2026-09-28T08:46:39Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104381379776090112","occurred_at":"2026-09-28T01:23:13Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","signature_id":"828834b6dd0e71a9b413e9e910f9937d0125a741c626c11c5bfe7e11833f2e4f","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104275998722756608","occurred_at":"2026-09-27T18:24:28Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104208646610096128","occurred_at":"2026-09-27T13:56:50Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104124987630686208","occurred_at":"2026-09-27T08:24:24Z","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103929841903472640","occurred_at":"2026-09-26T19:28:58Z","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881469456814080","occurred_at":"2026-09-26T16:16:45Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":17,"runs_affected":17,"failed_runs":17,"run_impact_percent":2.4045261669024045,"anchor_occurrences":17,"after_last_push_count":21,"also_in":["PROD","STG"],"trend_counts":[3,8,7,2,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":40,"prior_last_seen_at":"2026-09-24T12:25:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000721c20\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n      \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000721c20\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n      \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100c160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n      \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100c160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n      \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103098516569067520","occurred_at":"2026-09-24T12:25:34Z","signature_id":"ed02ea736a12b453c1efc5a8d4a86dc6bf8f47885f359a94f77192dc2ef4318b","pr_number":7065,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"569706336b0bb772a90b6de148018cee0d7066e56c9777657f620d7bb7c75db6","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","signature_id":"0c93d7531b51f1ea100516ccd01b0a1c6615ecf8a2bbdc599e9f78e9e71a81e6","pr_number":7020,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"88d7c43f7ba21a8263fb8b4c871ac116be74ea1e4b599156e118374c23dbf6bf","pr_number":7110,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","occurred_at":"2026-09-23T17:54:24Z","signature_id":"d95e88444841564bfebc7b4d86984dd9a38294a94ca9cf72a10f4adf98342abd","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","occurred_at":"2026-09-23T16:40:45Z","signature_id":"8a478233ef9f4819c24a5dc34485444f654a99c79dc829ccb980b0bc2625e398","pr_number":7114,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","occurred_at":"2026-09-23T13:26:24Z","signature_id":"e4012e513d3cafa3f6c8980450c912f0d288e7cf2e0c893549e7a66042041837","pr_number":7106,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","signature_id":"8eb0ca7714bb43bcafee5fa2084b2284a94351ac2160b6d55e893d478c2e48d8","pr_number":6439,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"a9f1409cf050d25886bb76839830611031e806e434f180cb1affd1b0a192d798","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102506260145377280","occurred_at":"2026-09-22T21:12:09Z","signature_id":"4bed88c5a61f4c358d0feca50612bfa11a44b660f96b61d2ab55333e0cc202f0","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102401634003849216","occurred_at":"2026-09-22T14:16:24Z","signature_id":"119f392f012953d3976e33cee72be8c3b09d267ec41ed4878feee3bb260cbeae","pr_number":7071,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367380872105984","occurred_at":"2026-09-22T12:00:18Z","signature_id":"9fc712ec1800f8f44b8fa79c7cd63de7cea995fbb405e3ab6f51fbabdcb7b683","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366747628670976","occurred_at":"2026-09-22T11:57:46Z","signature_id":"c5182d9a8aba3cc03529aeb383c02d293502a368a20ea2a1928a3c2c21a08902","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102322643775524864","occurred_at":"2026-09-22T09:02:32Z","signature_id":"cf31419f2da35385da91c35d1264050fd9b33627fcc4d2faa1e1ec321e4a403d","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","signature_id":"5331077f0794dfd9d86a1d07f697ecc5bb7dd9ee6da2ea69bbcff92a887a5cbf","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102283819666116608","occurred_at":"2026-09-22T06:28:15Z","signature_id":"b12375b84d62728be721797e6df5f1f2636a84e36bca55635e5688467a22cefb","pr_number":7041,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102246390443806720","occurred_at":"2026-09-22T03:59:31Z","signature_id":"cc4d49d2211113c6b9256ca01a1df7e2458e210a0e0b70c8dcf87367db634822","pr_number":7060,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":17,"runs_affected":17,"failed_runs":17,"run_impact_percent":2.4045261669024045,"anchor_occurrences":17,"after_last_push_count":43,"trend_counts":[0,0,0,15,2,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":84,"prior_last_seen_at":"2026-09-25T02:07:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":17}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T03:34:52Z\n  Ended:       2026-09-25T03:43:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7908608-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-78567cf5bf-qmbvj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-78567cf5bf-qmbvj (namespace kube-applier) on cluster ci01-j7908608-mgmt-1 has averaged 138.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T03:26:19Z\n  Ended:       2026-09-25T03:40:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7959296-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-7dc989b6d9-rg6jc\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-7dc989b6d9-rg6jc (namespace kube-applier) on cluster ci01-j7959296-mgmt-1 has averaged 138.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T01:15:33Z\n  Ended:       2026-09-25T01:24:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0596736-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-646d8d7d48-fmq9z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-646d8d7d48-fmq9z (namespace kube-applier) on cluster ci01-j0596736-mgmt-2 has averaged 129.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"40d5ad535cc1dad15220e6b19910e3eb9b33324cfc93904c2a5a3b66cda18505","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103304771547959296","occurred_at":"2026-09-25T02:05:09Z","signature_id":"bfb35c392292260a733e3e716557db76a69fa86926d85051968134ab2390355d","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270655540596736","occurred_at":"2026-09-24T23:49:35Z","signature_id":"cbcc41fb87ceb239baf4c483d3ab5268cdfd05b5c1837f310e5d12738e55e904","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","signature_id":"00ca63290ee1dd8cd9cc17fb467e5abff9ca540cb77527300f636b61f37a158b","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","signature_id":"31141dc7fe43f6d2aa89337a0a716ccfb12d368d6474df9ae0a1733b03fcf72c","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103245132269621248","occurred_at":"2026-09-24T22:08:10Z","signature_id":"cdf5e1de46d904b5754f4fcc48d70aa65ef0855075ecd7f8c6dfc039233eac7c","pr_number":7059,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"6a245a5382730698531a737265dac078b3a944a08cb0f737e3cfdc39f19615db","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"8d1d5c509bd74b9cfd90fd52435b1d0a29f7a99c0e5bc532798559b82949daa7","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","signature_id":"cedddb9dfe24b0472f0b69938e215469ad1fa1732808690eb588c2842038524b","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218872113958912","occurred_at":"2026-09-24T20:23:49Z","signature_id":"f448f1107e13285139f08f0b5d64b030989f1fce9c96e0627368c8e075d775c1","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"474887e104d72f6b5e1fe9db5422a8d481f6d2378182349a4ce3015e81b513f6","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203214361890816","occurred_at":"2026-09-24T19:21:25Z","signature_id":"855925e2df7033b23d85ecff7ede1ca6cb961907816303723c8e216cb000b37a","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200191157899264","occurred_at":"2026-09-24T19:09:35Z","signature_id":"784e6484d3fc29d5f43a3850fcf318afb800acb38164e1b18952339bbb4b08b9","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","signature_id":"b7cba4cec6f5a81a201092312fdb1ca862649a7f4b0539eae14accd98ac8dba5","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"c23483f9fa6966a89f6cf4d0f9d5416c765a770957ef669010321a77abc1e041","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","signature_id":"78083eff97a88f0d621458e0dfb940e0f79aaa0a1700d409ae56663484ef417b","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103175794774773760","occurred_at":"2026-09-24T17:32:39Z","signature_id":"e7b9c2ed622a10fb2e558d1618704e5e291864f9724b4b5ab5eb019d5bcadf46","pr_number":7053,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-24vw7c2tw2cd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-24vw7c2tw2cd, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":15,"runs_affected":14,"failed_runs":14,"run_impact_percent":1.9801980198019802,"anchor_occurrences":15,"after_last_push_count":11,"trend_counts":[0,0,2,1,6,1,1,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-27T08:18:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":11}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001512f00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-5f68mkqs6vtp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-5f68mkqs6vtp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001512f00\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-5f68mkqs6vtp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-5f68mkqs6vtp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0033b6750\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-nh4vk2wgfp5z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-nh4vk2wgfp5z, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0033b6750\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-nh4vk2wgfp5z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-nh4vk2wgfp5z, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc003167da0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-kg9lzmtgpr42\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-kg9lzmtgpr42, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc003167da0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-kg9lzmtgpr42\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-kg9lzmtgpr42, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104641856926126080","occurred_at":"2026-09-28T18:38:15Z","signature_id":"da523c4bd417aae8b3768777b0207e474c00bde3f1c27acfd3181f0c1268a499","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104397716221071360","occurred_at":"2026-09-28T02:28:07Z","signature_id":"99c1e28c6e269b6801651e5653d78953f7900215867405e5ac5b7f8e3eed035f","pr_number":7212,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","signature_id":"78fea1fb57019e1048fa9060e13e93e1530c79bf2bc64b62209db632f0720737","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","signature_id":"f20dd3825c25c084156a068aedceedf99a380bc52849b4182a4b8ef151834800","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104123485520400384","occurred_at":"2026-09-27T08:18:26Z","signature_id":"9b8aa1049a27cadfcf00f24cced762a2cb52851eacbb3b0c5ad6f8e5a3f78f90","pr_number":7193,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927280047755264","occurred_at":"2026-09-26T19:18:47Z","signature_id":"e3b07109b49bcafaeae2aeacce0e01aa2904db04b4015d601de9455d55fb2226","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103534322014228480","occurred_at":"2026-09-25T17:17:18Z","signature_id":"db842ed64a05de755e22b9021ea7593d77cdd91048ac8b40b56468fc1ec8705d","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103489841365782528","occurred_at":"2026-09-25T14:20:33Z","signature_id":"035d7dc1692505183a61563c7c881b11d2dac528da1fe3116304bc75371fc438","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"b09aada1d36d2ce5d689faef6eae5346e1bed7ccb7417af23f7078fe3705cf6e","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103457038066520064","occurred_at":"2026-09-25T12:10:12Z","signature_id":"a46ff91bb9b72d4ce7de86d214d11c025bf439cd09bdf0a909c8fdbf3a4d558f","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","signature_id":"145d4ba5cffaac551d0281b7858ed436f7d626d510091524d5c39f50edc9ea43","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","signature_id":"23feb3a96cb75059920f0d80a8340e60dd48c1eb6d7d1330a687410267280342","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"bf45d3a1f9dc91a9af5a7119dd1b60337e9f49919a6a81afba2cb9b9a6885627","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"19b631f7d5f2a850bd1bdac6e41017fbcf045a75d8474e6adf13f1324cd43c53","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","occurred_at":"2026-09-23T01:27:26Z","signature_id":"75beab8f572182a387939937ea4f447a02998cc8ddb2ad7397436d6df460ffc0","pr_number":7090,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":1.6973125884016973,"anchor_occurrences":12,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,0,12],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":5},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-28T22:45:44.302Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T22:53:20.758Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T22:45:44.312984171Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T22:45:46.293945727Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T22:45:46.372370612Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755330702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"93D44669EC2FD152\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT2M24.7661361S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755440055Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D5BB7C72C0AE0DC5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT2M26.0225935S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755461175Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3C41555BEADB3DE5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:45:49.7869316Z\\\",\\\"duration\\\":\\\"PT6M25.5435412S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755483916Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E73090DFF813B1FF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT1M41.3605417S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755497506Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DB7E7572BE861E73\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT1M37.1165773S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755594868Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A1922F5B4A5E438F\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:45:49.7869316Z\\\",\\\"duration\\\":\\\"PT5M2.9744241S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755630539Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109713389587325\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n        \\\\\\\"details\\\\\\\": [\\\\n          {\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\n            \\\\\\\"details\\\\\\\": [\\\\n              {\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\n              }\\\\n            ]\\\\n          }\\\\n        ]\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m36.442652668s\\\"}\\n exit status 1\"","time=2026-09-28T17:38:03.261Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T17:45:40.947Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T17:38:03.27081091Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T17:38:05.230722026Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T17:38:05.332101871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944777574Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"FDC79503A5389868\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M36.8422604S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\",\\\"target\\\":\\\"4\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 2, 1), 'Standard_E8ds_v6' (zones 1, 2), 'Standard_E8-4ds_v6' (zones 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944878586Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DC248B23802810EC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M22.2984424S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944892486Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F64F2B13FFA5C10A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT2M10.1227436S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944904226Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"061A995FDE246942\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:38:09.8797046Z\\\",\\\"duration\\\":\\\"PT6M25.5728844S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944914146Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"4E79D1B051092E6E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M21.7193107S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944924847Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E245F41D17E48B12\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:38:09.8797046Z\\\",\\\"duration\\\":\\\"PT5M2.6044727S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944945037Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109897999722695\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n        \\\\\\\"details\\\\\\\": [\\\\n          {\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 2, 1), 'Standard_E8ds_v6' (zones 1, 2), 'Standard_E8-4ds_v6' (zones 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\n            \\\\\\\"details\\\\\\\": [\\\\n              {\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"4\\\\\\\",\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\n              }\\\\n            ]\\\\n          }\\\\n        ]\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m37.674139897s\\\"}\\n exit status 1\"","time=2026-09-28T16:48:46.474Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T16:56:22.318Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T16:48:46.48313589Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T16:48:48.126456101Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T16:48:48.218251838Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316196451Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"466F265DD247AC33\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:51.5604197Z\\\",\\\"duration\\\":\\\"PT7M11.5958475S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316251615Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C2707792104BB37B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M38.4094883S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316265115Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"ED6706B9C7B9A085\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M54.5772258S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316275576Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9BFCDEDEF69AC484\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M20.8551291S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\",\\\"target\\\":\\\"3\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316323959Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A08358469F613804\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M37.4251788S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.31633432Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F3D7EC585ED8A035\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:51.5604197Z\\\",\\\"duration\\\":\\\"PT5M18.0784864S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316354711Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109927570685394\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n        \\\\\\\"details\\\\\\\": [\\\\n          {\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\n            \\\\\\\"details\\\\\\\": [\\\\n              {\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\n              }\\\\n            ]\\\\n          }\\\\n        ]\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m35.833223151s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104696906683781120","occurred_at":"2026-09-28T22:17:00Z","signature_id":"7f499ac06c32b370e2096e3b168803976b54da2a1f11dc54f9c1679c6f5bae1f","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104618768113078272","occurred_at":"2026-09-28T17:06:26Z","signature_id":"3cf8fbf66e63c9add929754f6f22e240822d57f521f0d283a39c4b61044dd3ef","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104608138077409280","occurred_at":"2026-09-28T16:24:16Z","signature_id":"f27f95b87b433141f34ec0551accb984fb054d19f57fbff11ef1f0f0955767f5","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605937833611264","occurred_at":"2026-09-28T16:15:31Z","signature_id":"e15b79721db530238062d8eb5258b0f475f5b9ee91938ada88d2f06997fe9fd9","pr_number":7074,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056927129600","occurred_at":"2026-09-28T15:20:22Z","signature_id":"245fc686bb329687b97bc133024cbedcbb44cd86f8d1e1e47b118dbf570d5a36","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056415424512","occurred_at":"2026-09-28T15:20:22Z","signature_id":"e12ab96347b90637e243fdb23adfad98d667a0a645d1910c2ec494ebefdb4f91","pr_number":7215,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104582134038532096","occurred_at":"2026-09-28T14:40:48Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104579857332572160","occurred_at":"2026-09-28T14:31:53Z","signature_id":"5b0cccc265d8272c63c2015099aee3246e496694b91725d91dd789992650d9f5","pr_number":7200,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576806710415360","occurred_at":"2026-09-28T14:19:39Z","signature_id":"596d124e078a20a425d61e9930296af8fedf5e0099dcc740de0f522cfa8668af","pr_number":7215,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104540479675699200","occurred_at":"2026-09-28T11:55:25Z","signature_id":"b2ae5eb0392e6e1bc7dcec48c107a273940a93afec94e0712cb5617629f457a9","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526841447452672","occurred_at":"2026-09-28T11:00:39Z","signature_id":"7ee3e6eb7ec864a3b373ec2c719dff573935d530dea03477c59c6bd3db3e8258","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104491225984798720","occurred_at":"2026-09-28T08:39:42Z","signature_id":"1ee68ae544fafb7bb55d51a213ba66048f990c8e725c3242ac72ba56bbd6192c","pr_number":7217,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":12,"runs_affected":12,"failed_runs":12,"run_impact_percent":1.6973125884016973,"anchor_occurrences":12,"after_last_push_count":17,"trend_counts":[4,3,2,1,3,1,1,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":31,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":12}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T21:44:52Z\n  Ended:       2026-09-28T22:03:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T21:44:52Z\n  Ended:       2026-09-28T22:00:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 12 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f3mdjcvo3r94pnej4b7fb4gsor4-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6fa9chvq085h1gvcth6so4h4lfhv-k2s0j1o5g8h6x7l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6irrha761ifugv0q9ve23paki69c-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-27T22:17:30Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6f5fj8gfr8k5odma9em9tq6i4f3j-f0z9g5b5n6v8h2l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-27T22:19:32Z\n  Ended:       2026-09-27T22:44:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6j7re0275kpgl4hqp81isfr27n57-r2q8i7j0v0z2u1y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:48:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-27T22:20:28Z\n  Ended:       2026-09-27T22:44:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0359424-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0359424/providers/microsoft.monitor/accounts/hcps-j0359424\", namespace=\"ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8zcr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t3k6kln64qe3bnhq9dg58qvhurq1ran-u8r4i9k8p5r5w8l/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T00:06:56Z\n  Ended:       2026-09-27T00:37:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T00:06:56Z\n  Ended:       2026-09-27T00:36:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","signature_id":"cb695e52ed5480bf4c59e72c73a827f71f03695861ee7b5fcc624983d0793694","pr_number":7229,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"e00c24569b7fef8b6c9f87bbc44d7468ad1c94ae60292dc7180f5c34825d2871","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0dfb7950c4b9435e9a2c892dc201219aa13f6c87439b016889682cade7a522c2","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","signature_id":"36613c51dc94f1ebf66f56c186a77e9398994f4128abf41aa5fe8abb52c4e2a8","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","signature_id":"9e87fb340216b923c3b3e1c2c19a207c8d191371d5639a1834a3d1a4fa9ac16f","pr_number":7174,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470957527044096","occurred_at":"2026-09-25T13:05:31Z","signature_id":"b3ccd27aced22d80e0151b4a766369a088986df8831ba4c138ed86573b3162d6","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","signature_id":"98313410c810095dec26c72ec7f4b43b7a09e0afa8d35824ba5fc5fec76fe165","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"d57782e3eb6f6cd2059f92437161a764f82ac19056eed7d813da72181ad2f92d","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","signature_id":"acc0593d70344c6ce5881c51325fa15f3ec8bcecedf6ebedda37a38bcf852bf6","pr_number":7077,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"a2c0bd06e35a7576d6d086fe8bc93742d833cb34906141aac4bb8ad9ab215f07","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","signature_id":"da3ffd0280a3f1ffcc889e6cdc2362c3329776d834e1983ae2fd4b9b85c57e98","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","signature_id":"c8a39895ab05da242fab6d0c9aa12843979da1c341ad1c2d1621b2491d4b33cd","pr_number":6516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":12,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":12,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,1,8,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012aa3f0\u003e: \n    GET https://rp.j2517632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\",\n      \"name\": \"df74e584-612a-4664-98e5-95894a223dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:13:31.817925786Z\",\n      \"endTime\": \"2026-09-28T21:42:34.60101187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012aa3f0\u003e: \n    GET https://rp.j2517632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\",\n      \"name\": \"df74e584-612a-4664-98e5-95894a223dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:13:31.817925786Z\",\n      \"endTime\": \"2026-09-28T21:42:34.60101187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de2a20\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tcmg5wddbpw5\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"name\": \"75ce2f03-ab4a-4df9-8666-ffa692e52e38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:31.92972128Z\",\n      \"endTime\": \"2026-09-27T22:26:36.766928511Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f767a0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-tlzxr8jmz7kk\" to finish creating: GET https://rp.j0359424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"name\": \"4b1edad4-5029-44ea-8a67-f8504dd1dd95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T21:57:15.95420119Z\",\n      \"endTime\": \"2026-09-27T22:26:17.083195861Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","signature_id":"bfb0b49b1e98aa126ca5cf25522dd823c2ee08043d22226794dfcfc1bbb69bcd","pr_number":7229,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"0170db780c4df4adf7bb8a89ac92e967f775ba081325c4cac8d09effb77a4c27","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"046677b799dbe94e2abdd61ad6c306ffc70f772329a705fd183eeddfe68167fe","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"056f5aac48261bf6a11c23178e7053c153d8a77c8cd07b47c6d8f44988ca9be3","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"14017b17f411363983778e1786bd28771dd63d57ce07d0aef5decd8c20e45c89","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"21db66633adda02d93dd92446231c93c6316598ee599dce2a1a985be29893e32","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"3edaa5633377ee32cc909ec3f497506e56f88cc918442b0177acc1481df58f76","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"8fc8ffdd4987b337260088b62ea7c21d838b773930ca05d6c0bce06df0b64bba","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104319966160359424","occurred_at":"2026-09-27T21:19:10Z","signature_id":"fe1e6109eb86fcb5bef47d66924fe8d64130a64ca2fed35ead60f414517a7f41","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"9d0706e3969af7c111fcca838b1e51305acfc48ec2465de31927959306c28e66","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","signature_id":"c3b9b0160dc46937aaa2f233272eae5b30c1b9f32ad52223e551b6235fac54c8","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","signature_id":"837f249c37264e4b948db37729027e6d876d161f3cbaadf548aa629a76406376","pr_number":7174,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.5558698727015559,"anchor_occurrences":11,"after_last_push_count":13,"trend_counts":[2,3,0,0,2,1,0,5],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":33,"prior_last_seen_at":"2026-09-26T19:18:41Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":11}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T21:47:52Z\n  Ended:       2026-09-28T22:02:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T20:21:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 5 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:12:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z8g6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:15:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:15:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:16:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:17:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z8g6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","signature_id":"e0f653efbaa66eeb75116383c05ff2fa3c1601a49c7ad496cd303b7011f3952a","pr_number":7229,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"d0d5fd846dafdaf7af7c9317c65a2b6639e493453a72d807fd729028c2663758","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"ef765ffa110519699ce0b4ae4b0309fac4eb0cda12a23778343f5d81fb18eed6","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"a5285cd524928261e15e6eab19f259927d80a331ee8b33927eebc0a295dec504","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"c0ba742d3b8df7101a372c1ade1a76e79f1270d7ffa16216e6dc2f557ede0a32","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"59859e98e366b5af518d04f59b67247bf6559ad517e1190c64eba0f09c96ceae","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103523744113758208","occurred_at":"2026-09-25T16:35:16Z","signature_id":"8d79d2e209e5007a23d684b8f03e30c1b2188d6188648ba6feeec72bedbded21","pr_number":7174,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"53d2828d48a06e30d3c8c55f9cf1eb466eebe159ecb450ed5d78e68b07926aec","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"fd2edebd87cfbeca482aaf300e4be69a630973901365509fd78112388229bda3","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"870d0b24a6bee5d59390ff5f28a0045b37e45b6c73b08247aec63cfac9af5bdf","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","signature_id":"9846f4c1be6a2dfdf4774339e5a3e0d83657d7196755eebca0d28ea4bac15bd9","pr_number":6516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.5558698727015559,"anchor_occurrences":11,"after_last_push_count":9,"trend_counts":[0,4,1,4,0,0,2,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":18,"prior_last_seen_at":"2026-09-27T12:48:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":11}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000676bd0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002177290\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ea9b70\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ea9b70\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104191454321577984","occurred_at":"2026-09-27T12:48:31Z","signature_id":"5926710b9fa5f78e3a02a2c8e840a39699891ffa25494a7619c4e511a8bddcd0","pr_number":7196,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7187/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104071407439712256","occurred_at":"2026-09-27T04:51:29Z","signature_id":"f0dd05bcedb68800765b2105c3e810ca2e4be249bca3b12d32e008a16b7acfdb","pr_number":7187,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103224244262408192","occurred_at":"2026-09-24T20:45:10Z","signature_id":"ecbec1ab9f9617f6b0594d4fbd470c36978a735ccd4e20c469402d74bd7943c7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078389232439296","occurred_at":"2026-09-24T11:05:35Z","signature_id":"90dd6825e45727c7dbdbd1c4c8cd0b287a1d641727762971db09d4ac1b8c9c2a","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015078914691072","occurred_at":"2026-09-24T06:54:00Z","signature_id":"423b6e69577f5d18ef882789710f57eff74dad24bb4c0c9c09ca525154f2add8","pr_number":7127,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"3f39acc6e073ba27c21beb42a91e286c5d52972e441b176f4734975709089b1e","pr_number":7126,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","signature_id":"80e228d9d0c46c7582a7eb683b187f1c62b13a5fdc9e873f52821f9b7629b50e","pr_number":7065,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","signature_id":"d868d40ddc424d3fed5c1fcfc795706e5f9bff04fe8acad2628a131a0fc96977","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102421926960959488","occurred_at":"2026-09-22T15:37:03Z","signature_id":"cdf31af69c9f22027040e78d440beeb427fcf496c338987b154029c97938d2cf","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408702790209536","occurred_at":"2026-09-22T14:44:25Z","signature_id":"f7c65b6efaad1f15cb61fc9b0be29ce6b45f3525dd78d215bd3a40750a28f9df","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102408218671058944","occurred_at":"2026-09-22T14:42:34Z","signature_id":"3981163203feb69b0a9b3885b33c6c2b7c20d0dd44d64a4aed81b04a298bef52","pr_number":7054,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":11,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.1315417256011315,"anchor_occurrences":11,"after_last_push_count":5,"trend_counts":[1,7,0,0,0,0,0,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000eb40f0\u003e: \n    resource group \"fips-enabled-bn5q2wwsf7rp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn5q2wwsf7rp\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n      \"name\": \"3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:05.789845252Z\",\n      \"endTime\": \"2026-09-28T09:54:14.850550256Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0hvkv6dq5otq5pkoot1lod893ptj still exists (deletion dispatched at 2026-09-28T09:30:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000eb40f0\u003e: \n    resource group \"fips-enabled-bn5q2wwsf7rp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn5q2wwsf7rp\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n      \"name\": \"3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:30:05.789845252Z\",\n      \"endTime\": \"2026-09-28T09:54:14.850550256Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0hvkv6dq5otq5pkoot1lod893ptj still exists (deletion dispatched at 2026-09-28T09:30:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001700288\u003e: \n    resource group \"private-keyvault-vfws486bsfgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vfws486bsfgl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n      \"name\": \"5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:29:57.435847003Z\",\n      \"endTime\": \"2026-09-28T09:54:04.986151784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u18j4sj43hhdfhhr4o61io72f8235 still exists (deletion dispatched at 2026-09-28T09:29:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001700288\u003e: \n    resource group \"private-keyvault-vfws486bsfgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vfws486bsfgl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n      \"name\": \"5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:29:57.435847003Z\",\n      \"endTime\": \"2026-09-28T09:54:04.986151784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u18j4sj43hhdfhhr4o61io72f8235 still exists (deletion dispatched at 2026-09-28T09:29:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00290a2b8\u003e: \n    resource group \"rg-candidate-5-0-8h76gpwqntfk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-5z7vd6\" in resourcegroup=\"rg-candidate-5-0-8h76gpwqntfk\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n      \"name\": \"cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:35:00.890511025Z\",\n      \"endTime\": \"2026-09-28T09:59:04.96675702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ao372vngnfec0pqf548ff4sjj still exists (deletion dispatched at 2026-09-28T09:35:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00290a2b8\u003e: \n    resource group \"rg-candidate-5-0-8h76gpwqntfk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-5z7vd6\" in resourcegroup=\"rg-candidate-5-0-8h76gpwqntfk\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n      \"name\": \"cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:35:00.890511025Z\",\n      \"endTime\": \"2026-09-28T09:59:04.96675702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ao372vngnfec0pqf548ff4sjj still exists (deletion dispatched at 2026-09-28T09:35:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"1dabd48da26bb8f586d188358c2a6b848886de5984190e3c86cb8de205e943bb","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"51d9fd31471a1c0d304e788b52fdca01b1faf0db3e9c8c998e6e19021aff1d43","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"62133fbc2a990af4d75df5f3308f03fabd68fc9126fb8e338e8a039115454435","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"80098528f82c953ebcec21eb40009d3ad625bc737f3cdf6d0e0316ea06ac992c","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","signature_id":"f950417357ce3a8a0c953a3e489cb0646ab86a213b08de387c8ba03089892d50","pr_number":6359,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102463947360702464","occurred_at":"2026-09-22T18:00:43Z","signature_id":"14e32cd9da960e8003053642ebd37f344e6211daa26684d90da6de88a269dbbe","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","signature_id":"4fbe849ec315aa759e6eaaa36007105f2abfc855ed8d77b0a75867781de33a52","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102367431941951488","occurred_at":"2026-09-22T12:00:30Z","signature_id":"dec9dde40f2f10efdc3071a06f16f401ec1f491f0a234e712f32edcadd53b386","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7031/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102329484735680512","occurred_at":"2026-09-22T09:29:43Z","signature_id":"53ed55054c50d93669e63983143c3a224f81ef6552fcf02580387d52aff966af","pr_number":7031,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102231517219524608","occurred_at":"2026-09-22T03:00:25Z","signature_id":"2e08a12fbaab745f1c9ad8675e56cc4e1764db69fce02141a843f86421385411","pr_number":6945,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102210924612947968","occurred_at":"2026-09-22T01:38:36Z","signature_id":"c8393314a20132a3e98fc6cd81c0dae2452cf2d8e1b2e7e97ae3de2436c44317","pr_number":7060,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":10,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":10,"after_last_push_count":3,"trend_counts":[0,0,0,0,3,0,0,7],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-25T14:23:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c241a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-c8jw5nmfkndx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\",\n      \"name\": \"4538241d-2021-40b7-8228-fb4052b1f808\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:37.948094059Z\",\n      \"endTime\": \"2026-09-28T19:39:45.568048903Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c241a0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-c8jw5nmfkndx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\",\n      \"name\": \"4538241d-2021-40b7-8228-fb4052b1f808\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:37.948094059Z\",\n      \"endTime\": \"2026-09-28T19:39:45.568048903Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--gd5x92vx9cnh/cluster-hs-pre-r2s4dx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd2480\u003e: \n    failed to create HCP cluster cluster-hs-pre-r2s4dx: failed waiting for cluster=\"cluster-hs-pre-r2s4dx\" in resourcegroup=\"rg-hs-pre--gd5x92vx9cnh\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n      \"name\": \"471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:54.032352928Z\",\n      \"endTime\": \"2026-09-28T13:23:00.490629254Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--gd5x92vx9cnh/cluster-hs-pre-r2s4dx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd2480\u003e: \n    failed to create HCP cluster cluster-hs-pre-r2s4dx: failed waiting for cluster=\"cluster-hs-pre-r2s4dx\" in resourcegroup=\"rg-hs-pre--gd5x92vx9cnh\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n      \"name\": \"471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:54.032352928Z\",\n      \"endTime\": \"2026-09-28T13:23:00.490629254Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dbafa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xh7nfd4qdxnv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\",\n      \"name\": \"10cf3d15-2823-4e15-9280-d4bdf793742a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:54:54.12824002Z\",\n      \"endTime\": \"2026-09-28T13:24:00.287293188Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dbafa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xh7nfd4qdxnv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\",\n      \"name\": \"10cf3d15-2823-4e15-9280-d4bdf793742a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:54:54.12824002Z\",\n      \"endTime\": \"2026-09-28T13:24:00.287293188Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"ae7ec42dfdec2a94af8e0db3707b53e37791d04766056ed14bd22a0a1022e2d0","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"601c5bd9f09b2e0f6ac00b5a321bb62c9488bb447cb61580b5be44db79b4f73d","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"6c6b92988197eb091819159a8cd6fffc7c5e8d6899de05bd4ab24749cc9fa574","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"bf243537c8da415f24304bc3723e10ad081d488b11614cdabdb3f6c9dd66a913","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"39d100c30a8f93dbf6247cda0ea2c5258c578cc34be4d24eed8741b80b0be4ac","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"560cf0a1b50bdf979e6f126cbc4861fee95b265a174c9f1c0ba41fcc25b22b09","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"e02b83b7b7d277f528654c1a7a21a3360f6f5084d7e91c17cd6c997d050b7856","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"043a7397f094ec76e0bf01b80760be01e69af512654f23e6e4da9739a3ccfa96","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"1934fc36111a36c3abccdd522caf0d3a5790367da6815ef25c43f7331a3c0e93","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103490540099080192","occurred_at":"2026-09-25T14:23:20Z","signature_id":"f1b09e484cd05c31b9ead39590e9ccaac7124eca4de0e096a44d8761ec49c147","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.272984441301273,"anchor_occurrences":9,"after_last_push_count":10,"trend_counts":[3,2,2,1,1,1,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-26T22:54:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":9}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T19:45:04Z\n  Ended:       2026-09-28T19:51:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"klusterlet-2t46re8cgt25tnc3e3r05ha4ola9p2m6\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t46re8cgt25tnc3e3r05ha4ola9p2m6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T05:30:55Z\n  Ended:       2026-09-28T05:36:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/services-j4374016\", namespace=\"klusterlet-2t3qakt150613r65up3s1cocvavjoksi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t3qakt150613r65up3s1cocvavjoksi/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T05:30:55Z\n  Ended:       2026-09-28T05:36:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/services-j4374016\", namespace=\"klusterlet-2t3qakt150613r65up3s1cocvavjoksi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t3qakt150613r65up3s1cocvavjoksi/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T00:21:19Z\n  Ended:       2026-09-27T00:27:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/services-j6084992\", namespace=\"klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7blf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t30mivbmcgkot8vhp8oa13ibv1vnaut/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"34ad39f09dd40559811aad8fffcaf6d1a0c0923be2148f882fd171659b3c1d97","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"d3a41ea55a2af8f7aa70bc89be3b1918a824c61a8f8ea40cec168f1ee7e1f5a8","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"a2eb7618d1e7a8cd04863ecb99aab31fa66480f2629953e2c4928c2aba4d4ae5","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"e1f51160fdcdfe2935875cac6be61e62b5d5cc074bef6b194f6075bfe9248be6","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6ffa9e2af62696b854cbaa4af470e9d2a538a83db7cca9ca00946bce22eef1d8","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"7f9ed0d0bd04a4b470fe9147f2b70a32e6b959bfcc09d740b923b169f280b98f","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"e210a4373a8a1dff8335c12bcbfb22a4bca9ffd1f73200c066c63547ac34373a","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"41bc348af7b09437b01801e49a5b4d2b4e19f50e7146ef23ebcde6ae3d0472ca","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","signature_id":"495668b1f8ea5dee71d0e8f473148935322994b6ef938eef0f10e2bfcaf03ca6","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.272984441301273,"anchor_occurrences":9,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[1,0,2,2,3,0,2,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-27T19:23:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000f972a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-t7rmzspt8s2k\" to finish creating: GET https://rp.j6124032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"name\": \"0aabc56d-9f58-4448-ac0c-c46df31d92cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T20:26:25.730010902Z\",\n      \"endTime\": \"2026-09-27T20:45:26.626992422Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-t7rmzspt8s2k--managed/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-t7rmzspt8s2k--managed/providers/Microsoft.Network/networkInterfaces/q2q3v2b3m8b4m9x-np-delete-main-7cgrv-l7n6k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '12eafc4e-f869-4b62-8198-816b7d6d0876' exceeded. Please try again after '10' seconds after additiona... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00089fe00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-25nvltvpwf84\" to finish creating: GET https://rp.j5652992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"name\": \"6c359dd9-afd9-4a97-8e2f-ad9a26c6937d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T17:01:59.070822701Z\",\n      \"endTime\": \"2026-09-27T17:21:03.436757642Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-25l2v-tmrtf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-25nvltvpwf84--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-25l2v-tmrtf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000adbee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qqrdr5ldkfxf\" to finish creating: GET https://rp.j2365184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n      \"name\": \"e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T20:55:06.599822055Z\",\n      \"endTime\": \"2026-09-25T21:14:14.361175634Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000adbee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qqrdr5ldkfxf\" to finish creating: GET https://rp.j2365184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n      \"name\": \"e499c0b9-b430-40af-9ef1-8fccc02c39c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T20:55:06.599822055Z\",\n      \"endTime\": \"2026-09-25T21:14:14.361175634Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qqrdr5ldkfxf--managed/providers/Microsoft.Compute/virtualMachines/j0r0n4d0n1i1i3m-np-delete-main-gskm4-2m8j2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104290856126124032","occurred_at":"2026-09-27T19:23:30Z","signature_id":"a7885b2a5379fa2f9992b69c22b756f6bcde94e94685985bf119094349859833","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7198/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104239747965652992","occurred_at":"2026-09-27T16:00:25Z","signature_id":"2d46f58172df00b26b08a908114c7df93ee1d42789e568fd90c7a7b65ba15046","pr_number":7198,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7172/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103574351302365184","occurred_at":"2026-09-25T19:56:22Z","signature_id":"d6768ec87762d85ec18639e021a53550ee7d398e2e064054a52f251bb3dee36d","pr_number":7172,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519758426050560","occurred_at":"2026-09-25T16:19:26Z","signature_id":"5cd60a99f0ee5798d016bc027d90478a36bcd3fdf3a9be5ae03b31e9803d6e8e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103424207261011968","occurred_at":"2026-09-25T09:59:45Z","signature_id":"878ef4ab647acb1943f7c9c33318d2974704d5a838a24ec4551ed0902a3582c8","pr_number":7003,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"84b235bf896d01e3deaa66856a2e9786768f1d93481789804dd05b50eee4738a","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","signature_id":"4de39a43d9c45741b51ae3bace1a381fb4260155eb8ea95a416b6fbf37a19a98","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"48a8490fb0e062a5d2a53db5bf3f5a92fd4adef796973aafbfa795c3fe0dbe08","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","signature_id":"0f9515a2db56a29e5a383d5fa728e947a2b0cee316b599fe636e25ce8c008556","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":9,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9900990099009901,"anchor_occurrences":9,"after_last_push_count":67,"trend_counts":[0,1,2,2,4,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":85,"prior_last_seen_at":"2026-09-25T12:24:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000990ae0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wf95j7vx69b5\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\",\n      \"name\": \"00381033-aeb7-400a-bd07-e1bd8611cea2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:15:51.082389682Z\",\n      \"endTime\": \"2026-09-25T13:45:00.023728639Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000990ae0\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wf95j7vx69b5\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00381033-aeb7-400a-bd07-e1bd8611cea2\",\n      \"name\": \"00381033-aeb7-400a-bd07-e1bd8611cea2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:15:51.082389682Z\",\n      \"endTime\": \"2026-09-25T13:45:00.023728639Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6ghnhlc94h8f/cluster-hs-pre-5wm67n should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f25a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-5wm67n: failed waiting for cluster=\"cluster-hs-pre-5wm67n\" in resourcegroup=\"rg-hs-pre--6ghnhlc94h8f\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\",\n      \"name\": \"46dab79f-2360-4680-914d-589d803cc911\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T05:47:00.77129988Z\",\n      \"endTime\": \"2026-09-25T06:16:03.97431147Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6ghnhlc94h8f/cluster-hs-pre-5wm67n should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f25a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-5wm67n: failed waiting for cluster=\"cluster-hs-pre-5wm67n\" in resourcegroup=\"rg-hs-pre--6ghnhlc94h8f\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/46dab79f-2360-4680-914d-589d803cc911\",\n      \"name\": \"46dab79f-2360-4680-914d-589d803cc911\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T05:47:00.77129988Z\",\n      \"endTime\": \"2026-09-25T06:16:03.97431147Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1aa00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgng9vz6j8w8\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\",\n      \"name\": \"a614d325-8937-4e4c-a91f-47b2564f0327\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T05:48:11.218096649Z\",\n      \"endTime\": \"2026-09-25T06:17:14.62086737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1aa00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgng9vz6j8w8\" to finish creating: GET https://rp.j9899776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a614d325-8937-4e4c-a91f-47b2564f0327\",\n      \"name\": \"a614d325-8937-4e4c-a91f-47b2564f0327\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T05:48:11.218096649Z\",\n      \"endTime\": \"2026-09-25T06:17:14.62086737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"080ef2ca65397bcb277ef88227d88a290162f457dffe508d8b1669032d36145a","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","signature_id":"27a55be51755e80b2d85191d7869a0f17115a878504d6f559e3291475acc6d1a","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","signature_id":"42a26a00c059eba85e0180aa4e4f83ccc2a993aadc3308bd54f4a077fbc7477d","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103346399469899776","occurred_at":"2026-09-25T04:50:34Z","signature_id":"5f77dd910e6de408d94f4725f867f2110f5e4dffba593f9da2e73cbb55e65e8c","pr_number":7157,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"2dd1d4adf03c4cf1ef2152b091a8d8db32bd11c08e0192a8877e14758cab2f4d","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"c29dc181d38e6ceb97183fd208302b380dff64e5d9a1ab9ca671c1309bb44d49","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","signature_id":"527306b3dde4b6e298931d98ad45937c8ba364a99fa4d7c4de2caf95207dcd18","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","occurred_at":"2026-09-23T16:20:35Z","signature_id":"2562b6a0aebd3dafa3041bae2bbef4d61a7d09096d6b568a3e7b98b9b1ca7da6","pr_number":7044,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102371297248940032","occurred_at":"2026-09-22T12:15:42Z","signature_id":"51057cb8671ceec45046c3798f134700b77132ac6c113e3e7197b692a3a31053","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.1315417256011315,"anchor_occurrences":8,"after_last_push_count":3,"trend_counts":[0,2,1,1,2,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-25T20:36:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-x7z7b7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bab020\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-x7z7b7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bab020\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c3c580\u003e: \n    DNS for route host agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c3c580\u003e: \n    DNS for route host agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002da48c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002da48c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-n4r5h.apps.aro.v20260901.fndt.j7290752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104433568867422208","occurred_at":"2026-09-28T04:50:35Z","signature_id":"cd7082ea03cbb08c0e4b8bdec829e02f4c6a4f777acb3eb5db34925ed981a912","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","signature_id":"0b9020620144e8fdaa405e64fc19ea1a76e527f3246863311db8886e6e3a419c","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103584488377290752","occurred_at":"2026-09-25T20:36:39Z","signature_id":"dd6c8bcde2b0624cac2879270fee01eca0a67b3bbceee5d0a34b90be6db52b16","pr_number":7104,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103555692374790144","occurred_at":"2026-09-25T18:42:13Z","signature_id":"6875b386161e5cce22e20953ae6d6d2de2afa405a8d3aec98377140ad121d02b","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","signature_id":"03a9333f9dd2535bd3ce94a401581227e326295231899b72504aa1c180c57341","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","occurred_at":"2026-09-23T20:12:38Z","signature_id":"5278202e490f8d8822dd976e22de925b821240a96d32e8bb0148f9031ab25a7c","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276754965008384","occurred_at":"2026-09-22T06:00:11Z","signature_id":"75ba3b9e29e1bbce0b25702e1769f68adca116986257d7f8084087bb33628dd3","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102276691408719872","occurred_at":"2026-09-22T05:59:56Z","signature_id":"efe8eb11fb7eb1e937c8ca8f32c5d4c2908dd8dfebec4f5bc55c17c0ba957c3c","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,1,7,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-24T09:33:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n      \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n      \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n    failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n      \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n    failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n      \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f01380\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n      \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f01380\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n      \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"648b322dd2b99b65149b365370cd6cb7208f3487446dfc081728eb5b19b547de","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"c83a023c6700a1a72267fb7de4ac02824e0ac5dfe0406b81242145d8b6fc24f6","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"dd690b63f5f18c2dbee25e78c6edf3eeb01364dc7ce7e0d6fe2be002e3e85b36","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"88da56dc9cbaf5632d307314826460b6bc87c0605368c5a80838d8aa0648f7a2","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"8bf81576a6cc6f8137c7015977c9d4e64ecd2342fb685d20f3e351e64bbf3b0b","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"e6dd7dea20380c21745a379f6c7147226e56bf52ffe41815559c0e6f0dbfefbd","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"f9b1fc76540f4c53a5a55de6e48304b8e30d158e9f8f3534a80d2ae685ba6b26","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","signature_id":"bccb01e525eebedfd6d8ea72bf2f185117e64e499bd98913c911974096d95997","pr_number":6439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":0.9900990099009901,"anchor_occurrences":7,"after_last_push_count":0,"trend_counts":[0,3,1,1,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-24T16:34:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":7}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T19:37:02Z\n  Ended:       2026-09-28T19:45:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T19:39:13Z\n  Ended:       2026-09-28T19:45:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-28T19:39:13Z\n  Ended:       2026-09-28T19:45:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-28T19:39:13Z\n  Ended:       2026-09-28T19:45:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:18:05Z\n  Ended:       2026-09-28T13:31:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-28T13:18:05Z\n  Ended:       2026-09-28T13:29:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:34:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:34:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"c7cb04f07a33dee0036fa4a43a64560ad19f5cf76e72a8e93d2645cb223f7e32","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"2cc7d70b704c84289c5bafb769a7557267d985237cbcee107436daf38e26c9c4","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"7726a18972655e8be5469370e6d72d78ec561de245d068ff794d9ec202b33f3d","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"80bbd3a02e54b8fc7971960324a0123e49b270a5d705be23d017d8e1a4b2f32e","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","signature_id":"a04e75c99557ace11f8b88485cfe566562bca31c71278e8a33d003f9d8dde804","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","signature_id":"624f8822ae56f0da81c94d2930d1e465c523cceb3e61be6edc365aee8089e257","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","signature_id":"fffa35f4057212200674fb96ca760632e8bcb5f7f0ce78e035f83a8a299a9866","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":6,"after_last_push_count":2,"trend_counts":[0,2,2,1,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-24T06:44:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000693320\u003e: \n    DNS for route host agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000693320\u003e: \n    DNS for route host agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b16880\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b16880\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551230880616448","occurred_at":"2026-09-28T12:38:08Z","signature_id":"d0a0ef141598400a6c63d0737ab5c55ee05359e8d50ed563cb531e14fff81ef2","pr_number":7106,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","signature_id":"c66a01613e5fe6442ebdb12c9ea63dd7c9f1f53d810adbdc9b51b498b061280c","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","signature_id":"0c29798c8c1659484795e61241cd7388e309cd731fe979947b39b617cfeb82bb","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","occurred_at":"2026-09-23T14:11:30Z","signature_id":"8c22befe458ceb42a7ccbeae384be919b61714799a47b6e0ea785ee3e8137ae4","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102448371858935808","occurred_at":"2026-09-22T17:15:09Z","signature_id":"eebf124ff7cd8e6579cd38d8f2821cfc541502c99089f74d0f5e0f453236134a","pr_number":4878,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102294129609281536","occurred_at":"2026-09-22T07:09:13Z","signature_id":"e8f2d1e264229b040ff7f19d77ed2f91221abe1ec550da844d4003af2d5756a9","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":6,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,2,1,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-27T04:49:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rxccfwgxq7jj\" to finish creating: GET https://rp.j1154560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n      \"name\": \"7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T06:32:25.841883321Z\",\n      \"endTime\": \"2026-09-28T06:51:27.808098854Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-hrvhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-rxccfwgxq7jj--managed/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rxccfwgxq7jj--managed/providers/Microsoft.Network/networkInterfaces/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rxccfwgxq7jj\" to finish creating: GET https://rp.j1154560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n      \"name\": \"7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T06:32:25.841883321Z\",\n      \"endTime\": \"2026-09-28T06:51:27.808098854Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-hrvhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-rxccfwgxq7jj--managed/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rxccfwgxq7jj--managed/providers/Microsoft.Network/networkInterfaces/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a2980\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-5fskdvpkxfwb\" to finish creating: GET https://rp.j7860224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"name\": \"2cbe44cd-36c6-4de6-8b35-e750b25ae85c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T05:48:02.556455592Z\",\n      \"endTime\": \"2026-09-27T06:07:09.276038359Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2: Failed: interfaces failed to create or update. err: failed to get existing resource admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-5fskdvpkxfwb--managed/providers/Microsoft.Network/networkInterfaces/t6c5p7s4s6o4n0f-worker-jlhpf-j8xj2-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceeded. Please try again after '10' seconds after... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b82600\u003e: \n    failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n      \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n      \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b82600\u003e: \n    failed to create NodePool np-wx6szx: failed waiting for nodepool=\"np-wx6szx\" for cluster \"cluster-candidate-4-20-wx6szx\" in resourcegroup=\"rg-candidate-4-20-r2kvmlfvs4fd\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n      \"name\": \"a8ae21a1-086f-493e-86bc-e120b31e75a1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:38:07.749745969Z\",\n      \"endTime\": \"2026-09-26T22:57:10.827139148Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-r2kvmlfvs4fd-candidate-wx6szx--managed/providers/Microsoft.Network/networkInterfaces/x5q9i7c1r3h9e2l-np-wx6szx-s4276-jbfxc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","signature_id":"f7628f13fed9510eb34469be7d0d678932294a06271b2a52d63d608fd70b72d1","pr_number":7202,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104070945927860224","occurred_at":"2026-09-27T04:49:39Z","signature_id":"ec4257f3d7d834907b3d94066f4a938edbf0824cf3b15c1781d4153ac05d1938","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"8a579545eb8ffc7ce43e3545cb26540ccd7358fcf1514bbca20793979fbd673e","pr_number":7185,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881459847663616","occurred_at":"2026-09-26T16:16:42Z","signature_id":"043f03e78369abc3dac495551386e55ba539c998c6f45575a66419b88c93eda0","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103487715302445056","occurred_at":"2026-09-25T14:12:06Z","signature_id":"5981f6c179bac3c28f9dc2c5df2c830c2d41379ed5d4dbea9adf66ba182514a6","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485377787465728","occurred_at":"2026-09-25T14:02:49Z","signature_id":"ca14cf116c18f677151c25e3f7d5bfbbd76e0c653dca506e7d80d70050fb3e24","pr_number":6920,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":6,"after_last_push_count":34,"also_in":["INT","PROD"],"trend_counts":[4,3,2,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":26,"prior_last_seen_at":"2026-09-25T13:05:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--nrk2cdj4tcn5/cluster-hs-pre-pcqks7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-pcqks7: failed waiting for cluster=\"cluster-hs-pre-pcqks7\" in resourcegroup=\"rg-hs-pre--nrk2cdj4tcn5\" to finish creating: GET https://rp.j7044096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\",\n      \"name\": \"e89da08a-794d-4885-a19f-3eb28678e2ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:47:52.911662059Z\",\n      \"endTime\": \"2026-09-25T14:16:59.018766333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--nrk2cdj4tcn5/cluster-hs-pre-pcqks7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009409a0\u003e: \n    failed to create HCP cluster cluster-hs-pre-pcqks7: failed waiting for cluster=\"cluster-hs-pre-pcqks7\" in resourcegroup=\"rg-hs-pre--nrk2cdj4tcn5\" to finish creating: GET https://rp.j7044096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e89da08a-794d-4885-a19f-3eb28678e2ea\",\n      \"name\": \"e89da08a-794d-4885-a19f-3eb28678e2ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:47:52.911662059Z\",\n      \"endTime\": \"2026-09-25T14:16:59.018766333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103470957527044096","occurred_at":"2026-09-25T13:05:31Z","signature_id":"3e7ca000732e0f699ce4bc50769e5e56cc9b3b001ef3b4dfb6ec429441e9f0d9","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"e0cff06d2d3890e9b05a9d35c28b2aff9003a5f93ca966b808d2a0e175a745b6","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","signature_id":"7f41eb165d596d5dc7f2971efb22eb9df93680423e7b77c55b04839a62820eb7","pr_number":7077,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"edd2ee36bfbf83bd2218ba79d6110db800244e83bcd1a31816119060e5229d1f","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","signature_id":"8dc69387aa9b7456b52151ccc6ab4f71148e68674b5c416f1655d1db2f47a7b6","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102321677739233280","occurred_at":"2026-09-22T08:58:41Z","signature_id":"fac8a1be8be2e9eccb68207302d9e3c8f8c385ae84412c13899620b63793cfe6","pr_number":6516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":6,"after_last_push_count":10,"trend_counts":[2,2,1,0,1,1,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-26T22:54:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":6}],"full_error_samples":["alert fired 16 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:33:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"cloud-network-config-controller-58dfb59986-bsjtm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2b4fc680-6a59-448e-b315-f78694553d9a\"\n  Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller-58dfb59986-bsjtm (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:33:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"cloud-network-config-controller-69985fdf59-fmd5d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"387b4461-aa28-4131-85f9-f2b77c4e2e0e\"\n  Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/cloud-network-config-controller-69985fdf59-fmd5d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:33:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"cloud-network-config-controller-69985fdf59-fmd5d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"387b4461-aa28-4131-85f9-f2b77c4e2e0e\"\n  Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/cloud-network-config-controller-69985fdf59-fmd5d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:33:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"csi-snapshot-controller-d4d6cd4f4-d2tqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c3f26a0c-8e66-4f75-bb2e-f2645071d798\"\n  Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/csi-snapshot-controller-d4d6cd4f4-d2tqr (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:33:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"csi-snapshot-controller-d4d6cd4f4-d2tqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c3f26a0c-8e66-4f75-bb2e-f2645071d798\"\n  Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/csi-snapshot-controller-d4d6cd4f4-d2tqr (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"csi-snapshot-controller-c9cf48cfd-vfz96\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e625ddc5-e22f-4d5c-bb2f-d4cb602de924\"\n  Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/csi-snapshot-controller-c9cf48cfd-vfz96 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"cloud-network-config-controller-5c87577d75-pzm4d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f7312b5e-d9af-4928-8d9a-ea34f1380f1e\"\n  Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller-5c87577d75-pzm4d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"cloud-network-config-controller-5b9c45d5bc-245tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e7400d-4c21-4a6d-a049-fcf41ebbe075\"\n  Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller-5b9c45d5bc-245tg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"cloud-network-config-controller-58dfb59986-bsjtm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2b4fc680-6a59-448e-b315-f78694553d9a\"\n  Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller-58dfb59986-bsjtm (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"csi-snapshot-controller-69d89876cf-6p78f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a54f46c-0195-4d5e-aae8-68c97f711f84\"\n  Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/csi-snapshot-controller-69d89876cf-6p78f (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"cloud-network-config-controller-5c87577d75-pzm4d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f7312b5e-d9af-4928-8d9a-ea34f1380f1e\"\n  Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller-5c87577d75-pzm4d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"csi-snapshot-controller-69d89876cf-6p78f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a54f46c-0195-4d5e-aae8-68c97f711f84\"\n  Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/csi-snapshot-controller-69d89876cf-6p78f (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"csi-snapshot-controller-7d6c8c6fbb-xmtc5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"07ef49a6-b3c7-427b-ab6f-44204ea64710\"\n  Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/csi-snapshot-controller-7d6c8c6fbb-xmtc5 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"csi-snapshot-controller-c9cf48cfd-vfz96\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e625ddc5-e22f-4d5c-bb2f-d4cb602de924\"\n  Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/csi-snapshot-controller-c9cf48cfd-vfz96 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"cloud-network-config-controller-5b9c45d5bc-245tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e7400d-4c21-4a6d-a049-fcf41ebbe075\"\n  Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller-5b9c45d5bc-245tg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T10:34:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"csi-snapshot-controller-7d6c8c6fbb-xmtc5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"07ef49a6-b3c7-427b-ab6f-44204ea64710\"\n  Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/csi-snapshot-controller-7d6c8c6fbb-xmtc5 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T00:09:50Z\n  Ended:       2026-09-27T00:37:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n  Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-27T00:09:50Z\n  Ended:       2026-09-27T00:37:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6084992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6084992/providers/microsoft.monitor/accounts/hcps-j6084992\", namespace=\"ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bb068423-1854-49c8-a132-0d10a6392326\"\n  Description: Pod ocm-arohcpci01-2t30mivbmcgkot8vhp8oa13ibv1vnaut-private-ingress/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T19:26:00Z\n  Ended:       2026-09-25T19:56:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fd1ced12-4634-450b-be94-fe821eb628d0\"\n  Description: Pod ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-25T19:26:00Z\n  Ended:       2026-09-25T19:55:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7879168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.83:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7879168/providers/microsoft.monitor/accounts/hcps-j7879168\", namespace=\"ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"fd1ced12-4634-450b-be94-fe821eb628d0\"\n  Description: Pod ocm-arohcpci01-2t27f8egsdoeqjvu0j3vd58dlko2an9h-r5q1e3i5h8g3j3f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"2723bca6220abf5c229937295d877dffcf70869f11654d5b10ba3a9c7d1e9f35","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103981635216084992","occurred_at":"2026-09-26T22:54:46Z","signature_id":"0e8d71bcb4b8b8f14ef36438303ce153649aa4d8d3bd9d130dfcadf1c764fbd1","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103549055647879168","occurred_at":"2026-09-25T18:15:51Z","signature_id":"d3f2c315f411e3fb04d632b66f8a302db0d803f5f870491f66c91c0c633ac840","pr_number":7168,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","signature_id":"d679a18a9e7a13bb82f0cbe4fdbabac5624367c05fd9b68e2584227db8962e2f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534053163962368","occurred_at":"2026-09-22T23:02:36Z","signature_id":"dc0ddfc8aae9b431fe2054ee903a37d2a737e09a536f135d1539226ec722ed8f","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","signature_id":"4d9ddf325a1d033b77f42eb5f81efe7623906a23240e70802d8b883ed1fe0a88","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":0.8486562942008486,"anchor_occurrences":6,"after_last_push_count":3,"trend_counts":[0,4,2,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-23T06:50:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:33:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j5147264-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j5147264-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:39:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j3407488-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j3407488-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T22:54:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j4395136-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j4395136-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","occurred_at":"2026-09-23T06:50:00Z","signature_id":"01cbd1d336e53f82a367140655d108e641e536db1311bf518e94e91e812f8a75","pr_number":6687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","signature_id":"7b5a0fe8e7d62a0661702614b741d3d10ab9bf69cb3d6aad1f5d3a0807f9d130","pr_number":7099,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503575954395136","occurred_at":"2026-09-22T21:01:29Z","signature_id":"89f393992f21000a05fbc5b394889210c7304fa2f585389c34870f2a013602ad","pr_number":5846,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102499706289524736","occurred_at":"2026-09-22T20:46:07Z","signature_id":"43accc06602d22f800522fb6e09d903ef020ad90c8e254c4ba9352cfc9345f98","pr_number":7064,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102409636605857792","occurred_at":"2026-09-22T14:48:11Z","signature_id":"14b3fcd8667cbbc2313076de16166cf53629f131635aa37c2c3fd5d3b1e80eb6","pr_number":6821,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102337463992193024","occurred_at":"2026-09-22T10:01:02Z","signature_id":"4d9dcd6dafc17dff069e71dad10c9a950f19ca6ce800ac8f8f06506ff5cdcc58","pr_number":7066,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,3,1,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-24T16:34:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:33:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j4604416-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j4604416-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0769280-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j0769280-svc has been unreachable for 15 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T00:04:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j3566464-svc\", component=\"monitoring-infrastructure\", region=\"centralus\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j3566464-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"679aa0117967b7ad541fd1b96fee91e26fcf0b44074834ec0f9774a9ce5ea7a4","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"0bff05a4b68cd34010dac9320f26f4a4adb189546a09d7a89930f18e9ffb6349","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","signature_id":"2dd3e964221f66c841fca4fdbecbd25f55231ce99032e40e07a49730aa63f949","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","signature_id":"8bbd752890c6930e820a263345bf9af1b4d7a0775a4b9b8de16c22d020962e5c","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","signature_id":"5fd4ab4b504fac9b9bbcbb453f7cec49945ab22d2c27be696660fbe03e2d4f86","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,3,1,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-24T16:34:17Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":5}],"full_error_samples":["time=2026-09-24T17:17:41.012Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:41.794Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:42.232Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:27:46.168Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:27:46.168Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:28:46.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:47.488Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.185Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:28:49.205Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:49.216Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:49.680Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:50.092Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.620Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.999Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.506Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.949Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:52.429Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:38:55.132Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:38:55.133Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:39:55.134Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.414Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:57.150Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:58.718Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:59.129Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:39:59.937Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:40:00.244Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:00.730Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.347Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:40:01.972Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","time=2026-09-23T01:45:47.799Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:48.667Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:49.334Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:55:54.330Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:55:54.331Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:56:54.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:55.777Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.630Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T01:56:57.690Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:57.691Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.299Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.862Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:56:59.423Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:57:00.012Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.364Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.910Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:57:01.546Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T02:07:04.175Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:07:04.175Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:08:04.175Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:05.466Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.382Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:07.843Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:08.593Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.447Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.853Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:10.521Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:12.134Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:15:47.989Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T02:15:47.990Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","time=2026-09-22T23:47:19.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T23:47:20.376Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T23:47:20.376Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:47:20.465Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T23:47:20.652Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T23:47:20.652Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:47:20.723Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:57:23.239Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T23:57:23.240Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-22T23:58:23.243Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-22T23:58:23.878Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-22T23:58:23.878Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.112Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-22T23:58:24.439Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.439Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:24.639Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-22T23:58:25.408Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:25.408Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:25.669Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:26.134Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T23:58:26.496Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-22T23:58:26.679Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:26.994Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:27.236Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-22T23:58:27.460Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-22T23:58:27.460Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-22T23:58:27.560Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T00:08:29.375Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T00:08:29.375Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T00:09:29.384Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T00:09:29.909Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j3566464-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:centralus role:hcp-underlay-ci01-j3566464-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:3739bfae-90c4-46a6-9e9a-788bc6da855b region:centralus secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#3739bfae-90c4-46a6-9e9a-788bc6da855b INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T00:09:29.909Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.097Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T00:09:30.328Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.328Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:30.580Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T00:09:31.341Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:31.341Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:31.535Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:31.879Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T00:09:32.169Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T00:09:32.570Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:32.841Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:33.156Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T00:09:33.376Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T00:09:33.376Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T00:09:33.452Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T00:17:19.905Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T00:17:19.906Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"09989fcafa95c4bca4519a71c6fa418fa3c6af4fbe8bd7b270ead898a41c8b5c","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","signature_id":"f020f9342f43cace837cc83b8952affa433a8c7410f5746e59533d769ed278b9","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102534072453566464","occurred_at":"2026-09-22T23:02:40Z","signature_id":"1f7aa719c9f65ad08c7354a49de1693f07f66c1401899d7f84a949b5b6228c2b","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102503516554661888","occurred_at":"2026-09-22T21:01:10Z","signature_id":"45b35ae85a60aea966255b483a545628aa8222b68318189a84fac3b010fd6be2","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102471925983875072","occurred_at":"2026-09-22T18:22:13Z","signature_id":"a4004bc2a8c2a270ffcbd9c390ca9dbe509655978183e489d387c818f6992f6e","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[0,0,1,4,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-24T21:23:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"9783002e2eb58bbef4bfb807305e4c98f54648869ece93a50ed71011664015e3","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","signature_id":"b9e992e4dedaba952b861c4393e5971bfc00933bdfd510734b71bce7ea9324b7","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","occurred_at":"2026-09-23T14:56:11Z","signature_id":"3812816f7f5de01152a0f4bc5ce516ed73308e2f6080e009a2d35e56b303c38c","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-57bkrhmpnrrx\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-57bkrhmpnrrx, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.7072135785007072,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,2,0,1,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-24T06:52:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400360\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400360\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001904960\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vhq6gmmfrw9m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vhq6gmmfrw9m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001904960\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vhq6gmmfrw9m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vhq6gmmfrw9m, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"ea6fad37e59eac223c8a29baca02fe524e66bda14470a0e205eea9919102b3f0","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"5c839c8ba3fa92aeabac7465b91fbcdcc11b953f6140a77174e182a2f4dccc18","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"5940ace73d01ab95b0fb2644669c5a69b3f53304c116e4dc03542a7fde89df81","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"457a5314345082c90ad67356518bb9d84bd8d68c40ec7946a1e92e67744e5389","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"b63c6b57c4e89b577bd5c804453853880b014ecdcd220586c0a9bbfec37a1675","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":5,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,1,0,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-26T19:18:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000bc80c0\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-2zhgtf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\",\n      \"name\": \"d382c657-1215-4712-a751-4d1a4f16b05f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:36:53.979369262Z\",\n      \"endTime\": \"2026-09-28T10:00:55.003547453Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ic75rjvskjgforinth6afhbjsc5 still exists (deletion dispatched at 2026-09-28T09:36:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000bc80c0\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-2zhgtf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\",\n      \"name\": \"d382c657-1215-4712-a751-4d1a4f16b05f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:36:53.979369262Z\",\n      \"endTime\": \"2026-09-28T10:00:55.003547453Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ic75rjvskjgforinth6afhbjsc5 still exists (deletion dispatched at 2026-09-28T09:36:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010b4108\u003e: \n    resource group \"rg-zstream-upgrade-4-21-n75m8pfcjsx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kjvjjk\" in resourcegroup=\"rg-zstream-upgrade-4-21-n75m8pfcjsx5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\",\n      \"name\": \"45231285-274d-43cd-98fa-1c2da5982b8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:36:34.341653398Z\",\n      \"endTime\": \"2026-09-28T10:00:35.039725799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0d23l1qdtpftb5407dlteafc6gfr still exists (deletion dispatched at 2026-09-28T09:36:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010b4108\u003e: \n    resource group \"rg-zstream-upgrade-4-21-n75m8pfcjsx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kjvjjk\" in resourcegroup=\"rg-zstream-upgrade-4-21-n75m8pfcjsx5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\",\n      \"name\": \"45231285-274d-43cd-98fa-1c2da5982b8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:36:34.341653398Z\",\n      \"endTime\": \"2026-09-28T10:00:35.039725799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0d23l1qdtpftb5407dlteafc6gfr still exists (deletion dispatched at 2026-09-28T09:36:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a32078\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-47qvqf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n      \"name\": \"e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:37:59.959232804Z\",\n      \"endTime\": \"2026-09-28T10:02:05.043620822Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ma6360plq8ebh66osj03lihjgik still exists (deletion dispatched at 2026-09-28T09:38:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a32078\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-47qvqf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n      \"name\": \"e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T09:37:59.959232804Z\",\n      \"endTime\": \"2026-09-28T10:02:05.043620822Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ma6360plq8ebh66osj03lihjgik still exists (deletion dispatched at 2026-09-28T09:38:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"5ebf9b2f69debdd5fcda540a2cf54489bef4a09bbbd816074578798dfb183457","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"d0ecd7d267b8b497361e55a6ad824a7f4015da5efc239b874ba4bc345a393ad9","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"d26e7881bdda81d57d491dda55f8a6f74790459a671e16de24eb0c2a3210b3d0","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"febfa4ad19c12a01fa699d939f8ef5e4e624851c9e2c5e88835d04e4a4a57859","pr_number":6900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103927255972450304","occurred_at":"2026-09-26T19:18:41Z","signature_id":"aa958e32e1c37b11d08d6ef5dd635ad8ce90eced3b3170eac46f774a812933cc","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["PROD","STG"],"trend_counts":[1,2,2,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-23T22:03:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":7108,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-23T22:58:15Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":4}],"full_error_samples":["time=2026-09-23T23:25:15.442Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T23:25:17.399Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T23:25:18.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9983616%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe\ntime=2026-09-23T23:27:49.191Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9983616/providers/Microsoft.Resources/deployments/72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe/operationStatuses/08584114009676534307\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-23T22:31:07.038Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T22:31:08.814Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T22:31:10.117Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8206336%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1\ntime=2026-09-23T22:34:11.270Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8206336/providers/Microsoft.Resources/deployments/cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1/operationStatuses/08584114042161978286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-23T21:07:35.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T21:07:36.837Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T21:07:38.387Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4221952%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee\ntime=2026-09-23T21:10:39.591Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4221952/providers/Microsoft.Resources/deployments/5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee/operationStatuses/08584114092280138717\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","occurred_at":"2026-09-23T22:58:15Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","occurred_at":"2026-09-23T22:01:19Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","occurred_at":"2026-09-23T20:49:32Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","occurred_at":"2026-09-23T20:14:08Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6733,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":4,"after_last_push_count":3,"also_in":["INT","STG"],"trend_counts":[0,0,0,1,3,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-25T12:24:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00148e780\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-l6zhdc\" in resourcegroup=\"rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n      \"name\": \"8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:29:34.053488486Z\",\n      \"endTime\": \"2026-09-25T13:48:39.892100532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00148e780\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-l6zhdc\" in resourcegroup=\"rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8\" to finish creating: GET https://rp.j5675648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n      \"name\": \"8325ef8e-dd78-4b1c-8d7e-a8b5d077aa5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:29:34.053488486Z\",\n      \"endTime\": \"2026-09-25T13:48:39.892100532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-l6zhdc-jh8xxsghsgf8-np-upgrade-l6-78bab067/providers/Microsoft.Compute/virtualMachines/f0d9h9y2l6m6d5u-npupgrade-4-21-f2rqb-8jjcb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca7e40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rjcjqxxsgq7f\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\",\n      \"name\": \"17378f74-366a-451c-a16d-8f6eab8588d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T12:45:45.224128185Z\",\n      \"endTime\": \"2026-09-25T13:04:46.064091322Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca7e40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rjcjqxxsgq7f\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17378f74-366a-451c-a16d-8f6eab8588d2\",\n      \"name\": \"17378f74-366a-451c-a16d-8f6eab8588d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T12:45:45.224128185Z\",\n      \"endTime\": \"2026-09-25T13:04:46.064091322Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-rjcjqxxsgq7f--managed/providers/Microsoft.Compute/virtualMachines/u6o3a4n8w6e1h9n-np-1-n26gj-tdhxm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001061ba0\u003e: \n    failed to create NodePool np-pqhwsk: failed waiting for nodepool=\"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\" in resourcegroup=\"rg-candidate-4-20-4hfcxgvg226p\" to finish creating: GET https://rp.j2644608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\",\n      \"name\": \"2ca27daa-447f-4425-a737-272daf2be568\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T04:05:18.601636409Z\",\n      \"endTime\": \"2026-09-25T04:24:23.185191796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001061ba0\u003e: \n    failed to create NodePool np-pqhwsk: failed waiting for nodepool=\"np-pqhwsk\" for cluster \"cluster-candidate-4-20-pqhwsk\" in resourcegroup=\"rg-candidate-4-20-4hfcxgvg226p\" to finish creating: GET https://rp.j2644608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2ca27daa-447f-4425-a737-272daf2be568\",\n      \"name\": \"2ca27daa-447f-4425-a737-272daf2be568\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T04:05:18.601636409Z\",\n      \"endTime\": \"2026-09-25T04:24:23.185191796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4hfcxgvg226p-candidate-pqhwsk--managed/providers/Microsoft.Compute/virtualMachines/j5f0t5a1f6v6f0d-np-pqhwsk-wxvcc-pbr8x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460627115675648","occurred_at":"2026-09-25T12:24:21Z","signature_id":"e27c2b8a568a4d0f9b1055a191452eb1394a405a2e80a4022d25f6917c8f9fba","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","signature_id":"101f712af098953d60298de1798c777e135600eee6b011a0c229833e88e62074","pr_number":7165,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318164392644608","occurred_at":"2026-09-25T02:58:22Z","signature_id":"fe76c6b3b8bcd7b776b70ad4fdbbcc743eda7378548d4371f57952d35a7644da","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162932127862784","occurred_at":"2026-09-24T16:41:32Z","signature_id":"71ad104d7c45702c68a9e44ecbe6eb8ff08e7116726d4744adcdde9ab6895dc5","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":4,"after_last_push_count":5,"trend_counts":[1,0,2,1,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T02:07:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-25T03:35:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-25T04:01:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7908608-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.146:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dq66f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:19:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-24T08:23:25Z\n  Ended:       2026-09-24T08:37:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:44:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:33:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"73c465500cc6b96cb01f7f0c6b77eae53537748367fdb4379569cc20c60be2b5","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"db9539a235d0c98a40af1909968de9adb2c176e0ca9c6ae7df7436f7869eb782","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","signature_id":"fbe2fd343d3a9197d314d9b85e39c50bd319b22385b4b05cd0ef79ae96f47f43","pr_number":6869,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","signature_id":"0b112a92c68becda58d472a3dbc17b517a7589cf579446aec092f675af664746","pr_number":7108,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.5657708628005658,"anchor_occurrences":4,"after_last_push_count":2,"trend_counts":[2,1,0,0,1,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-25T20:17:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002737518\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc002737518\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullhctmh8kwn4gw.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011dc00\u003e: \n    VerifyImagePulled(acrpullhctmh8kwn4gw.azurecr.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullhctmh8kwn4gw.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011dc00\u003e: \n    VerifyImagePulled(acrpullhctmh8kwn4gw.azurecr.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1f800\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1f800\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104634571122282496","occurred_at":"2026-09-28T18:08:52Z","signature_id":"59553741fb4c75e51213f6e718afb7cc45e4ae5d08010e66f0c43031bbb2a5f9","pr_number":7181,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104578384683077632","occurred_at":"2026-09-28T14:26:01Z","signature_id":"ac47319bea43a8ae82a11fa35dcd2a3b65d8f02c99e7937378a00807512c62aa","pr_number":7181,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","signature_id":"0c6454c08c8ccd6642c0993a10a72be6b940d4b7c10aa86a4dd03a639e166731","pr_number":7169,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102385490215833600","occurred_at":"2026-09-22T13:12:15Z","signature_id":"bc47151889f90300b321741d89b94df457be37cf4936f4c5a0af9551d05801fd","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T16:19:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c39160\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-01-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c39160\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-01-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4b500\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4b500\u003e: \n    failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","signature_id":"04c64a39622e30f071bc799f77f8bc41c83ad2872a342f4b19cf528cbde3c282","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","signature_id":"c8f8afa64d574b78b209e2a21b36f1feb7a2662df850c394333f89ba4e19dde5","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103396091608174592","occurred_at":"2026-09-25T08:08:02Z","signature_id":"89cda5b6732c3ef010bea1f11ee4a5639cba21b5c854e052517ec53280975d30","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7772a8fa7766108b00c98006e84756f50f97d8f9c7242d532de03696ce3d923c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code EventHubValidationErrorFound; detail message [BadRequest] Validation Errors found: Table does not exist","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T07:08:12Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":3}],"full_error_samples":["time=2026-09-25T07:28:19.460Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T07:28:19.869Z level=DEBUG+2 msg=\"Waiting for existing deployment to complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78\ntime=2026-09-25T07:28:19.869Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll previously-existing deployment: pre-existing deployment failed: {\\\"code\\\":\\\"DeploymentFailed\\\",\\\"details\\\":[{\\\"code\\\":\\\"EventHubValidationErrorFound\\\",\\\"message\\\":\\\"[BadRequest] Validation Errors found: Table does not exist\\\"}],\\\"message\\\":\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\"}\"","time=2026-09-25T04:41:01.572Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T04:41:01.944Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T04:41:02.776Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=be82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-kusto-us%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fbe82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7\ntime=2026-09-25T04:41:32.894Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/be82b2c4bc5e91c9fb05bb7a64319b66570f7844a6d32af17e112fe3dd39a4e7/operationStatuses/08584112956230068016\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"EventHubValidationErrorFound\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Validation Errors found: Table does not exist\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-25T02:39:19.860Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T02:39:20.257Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T02:39:20.970Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection deployment=67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-kusto-us%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78\ntime=2026-09-25T02:39:51.103Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/67437eb973c49795934fc79733f3a01017eeb2994048cb050f5f7bf549795e78/operationStatuses/08584113029248069745\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"EventHubValidationErrorFound\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"[BadRequest] Validation Errors found: Table does not exist\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103381037483560960","occurred_at":"2026-09-25T07:08:12Z","signature_id":"0d5e324b1f12b05a7f184d994200282ceecef7225105b3e1d8933bd3491b399c","pr_number":7146,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103341305806458880","occurred_at":"2026-09-25T04:30:16Z","signature_id":"eb3a097ae9ea09f1b87574d34137bdc9344882caed893e38ceda6a45fc8e1166","pr_number":7153,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7153/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103310372827041792","occurred_at":"2026-09-25T02:27:25Z","signature_id":"eb3a097ae9ea09f1b87574d34137bdc9344882caed893e38ceda6a45fc8e1166","pr_number":7153,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'.","search_query":"ERROR CODE: InvalidTemplate","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,2,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T02:58:34Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":3}],"full_error_samples":["time=2026-09-25T03:23:26.924Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-25T03:23:27.268Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-25T03:23:27.485Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/23db75d3c4a6f5fe32de7df35567f808137cc8acac2d401192c396cc43f1121e\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-24T23:55:56.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T23:55:56.789Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T23:55:57.168Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/23db75d3c4a6f5fe32de7df35567f808137cc8acac2d401192c396cc43f1121e\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-24T22:47:58.457Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T22:47:59.030Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T22:47:59.240Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/754de7cea577bd27c73251066594765f388be9580303d6f77fd3975785c55d9b\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318212358705152","occurred_at":"2026-09-25T02:58:34Z","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","pr_number":7146,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270905017798656","occurred_at":"2026-09-24T23:50:35Z","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","pr_number":7146,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103249876442681344","occurred_at":"2026-09-24T22:27:01Z","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","pr_number":7146,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-27T17:03:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:133]: failed to get admin REST config for cluster \"cluster-candidate-4-21-cscwdf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001673c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-21-cscwdf\" in resourcegroup=\"rg-candidate-4-21-xbwd2trtds6r\" to finish getting creds: GET https://rp.j6653824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75bf6e3d-ec48-419a-a251-e8975a2853cf\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:136]: failed to get admin REST config for cluster \"cluster-zstream-4-21-kz5vz4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010728c0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-kz5vz4\" in resourcegroup=\"rg-zstream-upgrade-4-21-xtgkx9mrrcpz\" to finish getting creds: GET https://rp.j0275968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:136]: failed to get admin REST config for cluster \"cluster-zstream-4-21-kz5vz4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010728c0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-kz5vz4\" in resourcegroup=\"rg-zstream-upgrade-4-21-xtgkx9mrrcpz\" to finish getting creds: GET https://rp.j0275968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/773f5d4a-cd7a-4e25-ac56-58130cbed2d2\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:329]: failed to get admin REST config for cluster np-noedge-7z925l\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00077d780\u003e: \n    failed waiting for hcpCluster=\"np-noedge-7z925l\" in resourcegroup=\"rg-np-noedge-7z925l-xwhx4zwbzvsx\" to finish getting creds: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:329]: failed to get admin REST config for cluster np-noedge-7z925l\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00077d780\u003e: \n    failed waiting for hcpCluster=\"np-noedge-7z925l\" in resourcegroup=\"rg-np-noedge-7z925l-xwhx4zwbzvsx\" to finish getting creds: GET https://rp.j7511040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ac55f4d-25b7-4531-81f3-de46161013d9\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104255677336653824","occurred_at":"2026-09-27T17:03:43Z","signature_id":"05ebaa9488bd93eeb90e405c068beae1c93f0836a7ac6badc62d7e0b3eadd4c5","pr_number":7183,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","signature_id":"1b01d9e119adc9dde05f5270a026a56addb171a7b99ea7686bbcfe2a7ca2162d","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102353447767511040","occurred_at":"2026-09-22T11:04:56Z","signature_id":"3bb4ede6806075a37b8c28d7b2a38f450e3de8181d81d31103a281b5545686b9","pr_number":6092,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["PROD","STG"],"trend_counts":[0,0,1,2,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-24T12:28:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n    \u003c*errors.StatusError | 0xc0012d1400\u003e: \n    Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n    \u003c*errors.StatusError | 0xc0012d1400\u003e: \n    Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n    Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n    Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099155617419264","occurred_at":"2026-09-24T12:28:06Z","signature_id":"d8b50df50945eaa6be746f392a278fe85b30b5ed045a7c3106d19fdebdab862c","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"28954fd4b540505310c8d2b6c0bea86f4588ec03f99fb14067897e28bef073a8","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","signature_id":"23493215ee1a820f0efaa9de36d3f924d4bf22faf75dcf25d674d9fd4bee714f","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,1,0,1,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T19:17:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001afc030\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001afc030\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kzc9z.apps.aro.cilium-cluster.oaay.j3069568.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00268e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00268e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc001514030\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc001514030\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-ctpxj.apps.aro.v4m3f6y4q7z0t3b.chi6.j5582976.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103564467513069568","occurred_at":"2026-09-25T19:17:05Z","signature_id":"7e94a80a5cfc3c6e5e5e2149f42307fb2e5227e9fb4287e4913bf9419962a984","pr_number":7176,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","signature_id":"5b25ef7df46295d6b0322467710d77611928e860a415a5c20e227a626a8235c7","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102391598355582976","occurred_at":"2026-09-22T13:36:14Z","signature_id":"312f92f173f393934c2af6c3b15128820da22eb3a757e3a667a9dabf4eb5433b","pr_number":7077,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,2,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-25T10:20:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-25T11:11:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7570432-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T19:45:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j6380032-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T16:29:24Z\n  Ended:       2026-09-24T16:38:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1695232-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103429520567570432","occurred_at":"2026-09-25T10:20:52Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":7164,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,2,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T02:07:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-25T04:00:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j7908608-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7908608/providers/microsoft.monitor/accounts/services-j7908608\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j7908608-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T17:38:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j4475392-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/services-j4475392\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j4475392-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T14:32:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j7129984-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7129984/providers/microsoft.monitor/accounts/services-j7129984\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of completed cluster create (install) operations on ci01-j7129984-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305281587908608","occurred_at":"2026-09-25T02:07:11Z","signature_id":"0a3b060ab5af73558d8c0a74ad7ce0917f462e5e9705c25366e434181ee85d37","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"e94d5731f1171ed8ccd1a80a6bd7d03994a9e3251c8ef2f3a3e39c2a85a34276","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"498fd6b520c3adb9da74466053b7266c919395d71a055e8926fce3e947cf3a23","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.4243281471004243,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[1,0,2,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-23T17:52:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.008s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001488a20\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:46491-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.008s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001488a20\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:46491-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500002637221888","occurred_at":"2026-09-28T09:14:19Z","signature_id":"620340dd365edadda3dbd85304c7c7cde65b3e17f1f0272584c0a1bf12bb3b26","pr_number":7170,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","occurred_at":"2026-09-23T17:52:29Z","signature_id":"b269b2a196f9ad9cd1702b095e9f309573b62770b7cab43403d5312cc9615be1","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","signature_id":"312be1f9783180426404eca8136660afa7c8940a066aacb369c351024c0827a4","pr_number":7103,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000911660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-6qt82pv5cb4k\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\",\n      \"name\": \"cd024c78-818b-4b49-8634-2f704d9a8088\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T07:18:28.191649845Z\",\n      \"endTime\": \"2026-09-28T07:47:33.272794005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000911660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-6qt82pv5cb4k\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\",\n      \"name\": \"cd024c78-818b-4b49-8634-2f704d9a8088\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T07:18:28.191649845Z\",\n      \"endTime\": \"2026-09-28T07:47:33.272794005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001698220\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hqr7q7xq4llr\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n      \"name\": \"e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:08.942854859Z\",\n      \"endTime\": \"2026-09-28T05:23:14.16072165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001698220\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hqr7q7xq4llr\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n      \"name\": \"e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:08.942854859Z\",\n      \"endTime\": \"2026-09-28T05:23:14.16072165Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d2ce0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sbtnmq6dk9ch\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\",\n      \"name\": \"1f13de10-f443-42eb-b391-6417a9194758\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:55:46.663436186Z\",\n      \"endTime\": \"2026-09-28T05:24:54.296612042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d2ce0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sbtnmq6dk9ch\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\",\n      \"name\": \"1f13de10-f443-42eb-b391-6417a9194758\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:55:46.663436186Z\",\n      \"endTime\": \"2026-09-28T05:24:54.296612042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"0b89bdc7383f2c123c8287a1a5aec43b3c515722ab8e7d58f23bb0002aec1e93","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"9c95fddd44e42f0842cd3dae63f79a607777a45781192a59b8e078313504b0a8","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"dcbe953ab01aef145b8f85e06d22bffa5a686320c14b765bf9e95bf552ee460e","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,3],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a6bae0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5w2hj7t2fgmv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n      \"name\": \"fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:32.768656716Z\",\n      \"endTime\": \"2026-09-28T13:22:40.579367905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m26s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a6bae0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5w2hj7t2fgmv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n      \"name\": \"fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:32.768656716Z\",\n      \"endTime\": \"2026-09-28T13:22:40.579367905Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m26s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea49c0\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-q4hdh7csw5qs\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\",\n      \"name\": \"daab790a-4984-45ab-a2a3-a3f37cd79508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:48.885600114Z\",\n      \"endTime\": \"2026-09-28T13:22:50.894287651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m10s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea49c0\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-q4hdh7csw5qs\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\",\n      \"name\": \"daab790a-4984-45ab-a2a3-a3f37cd79508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:48.885600114Z\",\n      \"endTime\": \"2026-09-28T13:22:50.894287651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m10s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d60180\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-txbk9bmdh8wg\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n      \"name\": \"7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:22.508354202Z\",\n      \"endTime\": \"2026-09-28T13:22:30.229773982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 26m20s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d60180\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-txbk9bmdh8wg\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n      \"name\": \"7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:22.508354202Z\",\n      \"endTime\": \"2026-09-28T13:22:30.229773982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 26m20s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"4db5797f215c217305fb770b8ae20d1fadd3857acb12459b9d26de91ec6863e5","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"664087fc7585f829fbe7a6c474f2c0e06ccf5f5a842a67cc4a4c2cf5f9ca990d","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"e7153cfe927d09bf871348873ff5d0a76b6b72889ee031a3643abac543c6012d","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-24T09:33:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n  \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n  \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n  \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n  \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n  \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n  \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hb6mtzv8hm6q\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\",\n  \"name\": \"b4e55441-95a5-466c-adac-c574d8bcdb81\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-22T07:34:05.103384159Z\",\n  \"endTime\": \"2026-09-22T07:53:06.353871266Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-hb6mtzv8hm6q\" to finish creating: GET https://rp.j5260288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b4e55441-95a5-466c-adac-c574d8bcdb81\",\n  \"name\": \"b4e55441-95a5-466c-adac-c574d8bcdb81\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-22T07:34:05.103384159Z\",\n  \"endTime\": \"2026-09-22T07:53:06.353871266Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"node pool creation did not complete before the deadline\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102286205365260288","occurred_at":"2026-09-22T06:37:44Z","signature_id":"663948fad2f655a8f86778686b2675b46d2a717e3dc1afdecbbc520028b3d155","pr_number":6995,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-24T11:45:17Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2}],"full_error_samples":["time=2026-09-24T12:31:58.312Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T12:32:36.173Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json --to-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json\\n exit status 1\"","time=2026-09-24T10:30:31.503Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T10:31:04.673Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103088386418937856","occurred_at":"2026-09-24T11:45:17Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103060333542248448","occurred_at":"2026-09-24T09:53:51Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6f6e7e66ad8bc42bd6647f1e246b1e77205441e0c9495974d9b1f940d8bdec9c","failure_pattern":"ERROR CODE: Conflict; detail code ObjectIsDeletedButRecoverable","search_query":"ERROR CODE: Conflict","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[6,2,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-22T02:42:18Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/admin-api-cert","occurrences":2}],"full_error_samples":["time=2026-09-22T03:15:16.448Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-22T03:15:16.448Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j2691712 issuer=Self commonName=admin.j2691712.hcpsvc.osadev.cloud san=admin.j2691712.hcpsvc.osadev.cloud\ntime=2026-09-22T03:15:17.222Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j2691712\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j2691712/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j2691712 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-22T00:26:51.703Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert description=\"Step admin-api-cert\\n Kind: CreateCertificate\\n\"\ntime=2026-09-22T00:26:51.703Z level=INFO msg=\"Creating certificate\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert vaultBaseUrl=https://aro-hcp-dev-svc-kv.vault.azure.net/ certificateName=admin-api-cert-ci01-j1796608 issuer=Self commonName=admin.j1796608.hcpsvc.osadev.cloud san=admin.j1796608.hcpsvc.osadev.cloud\ntime=2026-09-22T00:26:52.483Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=admin-api-cert err=\"error running Create Certificate Step: failed to create certificate \\\"admin-api-cert-ci01-j1796608\\\" in vault \\\"https://aro-hcp-dev-svc-kv.vault.azure.net/\\\": POST https://aro-hcp-dev-svc-kv.vault.azure.net/certificates/admin-api-cert-ci01-j1796608/create\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Certificate admin-api-cert-ci01-j1796608 is currently in a deleted but recoverable state, and its name cannot be reused; in this state, the certificate can only be recovered or purged.\\\",\\n    \\\"innererror\\\": {\\n      \\\"code\\\": \\\"ObjectIsDeletedButRecoverable\\\"\\n    }\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7062/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102226956232691712","occurred_at":"2026-09-22T02:42:18Z","signature_id":"d0835b4a87efe485e5d2632604de682cfdfca8dd7d942ad8c07295231be37bc7","pr_number":7062,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7042/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186818521796608","occurred_at":"2026-09-22T00:02:48Z","signature_id":"95e7be5f64e9ad50a9e7306687aca3408d8573d35d0eb9459a8bc2935088f19f","pr_number":7042,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"50d71e86102fe885def0430fc65ae5eda7365e37815a984b65ed42e4e299b068","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T18:09:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138a980\u003e: \n    failed to create HCP cluster np-dg-minor-w2lmrh: failed waiting for cluster=\"np-dg-minor-w2lmrh\" in resourcegroup=\"rg-np-dg-minor-w2lmrh-zzht7hmgmswz\" to finish creating: GET https://rp.j2515584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n      \"name\": \"8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T10:35:16.490368834Z\",\n      \"endTime\": \"2026-09-28T11:04:17.550861433Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 1m2s ago; version 4.22.16 is Partial (want Completed), started 7m32s ago; version 4.22.16 is Partial (want Completed), started 11m2s ago; version 4.22.16 is Partial (want Completed), started 18m25s ago; version 4.22.16 is Partial (want Completed), started 20m51s ago; version 4.22.16 is Partial (want Completed), started 23m2s ago; version 4.22.16 is Partial (want Completed), started 26m19s ago; version 4.22.16 is Partial (want Completed), started 27m32s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138a980\u003e: \n    failed to create HCP cluster np-dg-minor-w2lmrh: failed waiting for cluster=\"np-dg-minor-w2lmrh\" in resourcegroup=\"rg-np-dg-minor-w2lmrh-zzht7hmgmswz\" to finish creating: GET https://rp.j2515584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n      \"name\": \"8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T10:35:16.490368834Z\",\n      \"endTime\": \"2026-09-28T11:04:17.550861433Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 1m2s ago; version 4.22.16 is Partial (want Completed), started 7m32s ago; version 4.22.16 is Partial (want Completed), started 11m2s ago; version 4.22.16 is Partial (want Completed), started 18m25s ago; version 4.22.16 is Partial (want Completed), started 20m51s ago; version 4.22.16 is Partial (want Completed), started 23m2s ago; version 4.22.16 is Partial (want Completed), started 26m19s ago; version 4.22.16 is Partial (want Completed), started 27m32s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1a600\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-sm4jtnxclsxs\" to finish creating: GET https://rp.j5868928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n      \"name\": \"6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T19:01:13.53147986Z\",\n      \"endTime\": \"2026-09-25T19:30:14.113086545Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 4m17s ago; version 4.21.35 is Partial (want Completed), started 5m48s ago; version 4.21.35 is Partial (want Completed), started 13m55s ago; version 4.21.35 is Partial (want Completed), started 15m34s ago; version 4.21.35 is Partial (want Completed), started 19m39s ago; version 4.21.35 is Partial (want Completed), started 23m46s ago; version 4.21.35 is Partial (want Completed), started 24m59s ago; version 4.21.35 is Partial (want Completed), started 25m33s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1a600\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-sm4jtnxclsxs\" to finish creating: GET https://rp.j5868928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n      \"name\": \"6d34f713-a16b-433b-99d2-1c7b278e6b21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T19:01:13.53147986Z\",\n      \"endTime\": \"2026-09-25T19:30:14.113086545Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 4m17s ago; version 4.21.35 is Partial (want Completed), started 5m48s ago; version 4.21.35 is Partial (want Completed), started 13m55s ago; version 4.21.35 is Partial (want Completed), started 15m34s ago; version 4.21.35 is Partial (want Completed), started 19m39s ago; version 4.21.35 is Partial (want Completed), started 23m46s ago; version 4.21.35 is Partial (want Completed), started 24m59s ago; version 4.21.35 is Partial (want Completed), started 25m33s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","signature_id":"af5dba612f5bcafa125e14121eae54d6911f7b73c83a90c9caccd95170639937","pr_number":7203,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547336255868928","occurred_at":"2026-09-25T18:09:01Z","signature_id":"06d4cbae5cb178dd7474a8e93fa31108efbbc06d8bf341e39e73e3837d942123","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cd0b0b8f7ac9ea6402ca4eac6d805c330783a014b8096e0958d5137c2c282bdb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[3,2,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-22T10:04:32Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-09-22T10:35:24.276Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T10:42:31.669Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T10:35:24.282791483Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T10:35:26.284841392Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T10:35:26.448708771Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666852933Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"48630ADF8315F9F4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT1M40.6270261S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666918654Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"50E367DC062B740B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT1M39.3479577S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666940194Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"96EF32884A17F314\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT4.1058994S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.666978395Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1E29754134C2A673\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:35:30.2867656Z\\\",\\\"duration\\\":\\\"PT5M11.8733849S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667019736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DD00B42016EFC1CA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:40:38.6323694Z\\\",\\\"duration\\\":\\\"PT3.4027868S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667063427Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3962D5B17452A0C8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:35:30.2867656Z\\\",\\\"duration\\\":\\\"PT5M8.2796787S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:42:31.667099327Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2444800-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2444800-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115335588388698\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2444800-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2444800-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m7.384317605s\\\"}\\n exit status 1\"","time=2026-09-22T10:31:08.961Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T10:38:47.425Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T10:31:08.979907747Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T10:31:11.06396002Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T10:31:11.22661971Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423846978Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F61EEEEBCFA56868\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT1M39.7596603S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423898879Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"8E07C9049ECB8DEF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT1M39.7909151S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423911859Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C8376C88B6FADF35\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT4.063355S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.42394617Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2DFDEDAE29CB0E2C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:36:41.5611028Z\\\",\\\"duration\\\":\\\"PT3.9227798S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.423968521Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"04012E211BF318FE\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T10:31:16.3138337Z\\\",\\\"duration\\\":\\\"PT5M28.8419351S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.424009742Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F96D9B5A6875A606\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T10:31:16.3138337Z\\\",\\\"duration\\\":\\\"PT5M25.0918247S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T10:38:47.424030272Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4070272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4070272-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115338140513340\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft2\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      },\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4070272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4070272-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 18 for family StandardDdsv6Family for region canadacentral. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m38.444136305s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102338247882444800","occurred_at":"2026-09-22T10:04:32Z","signature_id":"3096746f308f4ed95275075880d60cb754e454faacf298548b21a679d504f7a7","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102333755334070272","occurred_at":"2026-09-22T09:46:41Z","signature_id":"56583212f49aa2bec3088ae2a65589f7c4f6c467661d1f7190268818dac419c1","pr_number":6439,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-23T23:42:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","occurred_at":"2026-09-23T23:42:18Z","signature_id":"9d20053e21637cc884e3f5b506bcfcb59a5c2cd22b2f8ee5ff24cb97bfeb11c4","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","occurred_at":"2026-09-23T18:09:20Z","signature_id":"a7690e671288365fae42a54239a1fd3d4683bfffda4c651366b6b6d63b190ce6","pr_number":7118,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-22T15:46:19Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T17:14:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j4475392-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4475392/providers/microsoft.monitor/accounts/services-j4475392\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j4475392-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-22T14:07:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j7129984-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7129984/providers/microsoft.monitor/accounts/services-j7129984\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n  Description: More than 72% of completed cluster create (install) operations on ci01-j7129984-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102424261724475392","occurred_at":"2026-09-22T15:46:19Z","signature_id":"32e03851f37111fa5a94d086994b49ba0284bc78df8fdff2b940cf9240caab63","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"4b82640645f0248147b9da24bd012fa2cd836d5c931b44904d7cc22a4314f873","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T12:24:19Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-25T13:12:26.954Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-09-25T13:12:27.437Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-25T13:12:27.438Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:12:27.610Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-09-25T13:12:27.765Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-09-25T13:12:27.765Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:12:27.816Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-09-25T13:13:20.164Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-25T13:13:20.164Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","time=2026-09-24T23:38:34.810Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.513Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.755Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103460622090899456","occurred_at":"2026-09-25T12:24:19Z","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103259159142076416","occurred_at":"2026-09-24T23:03:54Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.2828854314002829,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T02:58:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb6980\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb6980\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa2920\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-x4kl262s2lph\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa2920\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-x4kl262s2lph\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"e69c1d42b103c2fc2619e82648d81646d2d6df78f72c3c1d170576fb995e0bdb","pr_number":7124,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","signature_id":"c493dd3074bde168273260831e1897caadd72a13f32f14f010c37424a8cb9ea8","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"257b448a58141006c11f51362759eea79da6b0398afa6beef61e4326de15f4d6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2s722b\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfeb60\u003e: \n    failed to create HCP cluster np-noedge-2s722b: failed waiting for cluster=\"np-noedge-2s722b\" in resourcegroup=\"rg-np-noedge-2s722b-ttwnzxstsj4p\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\",\n      \"name\": \"18b3752b-dfe7-413c-83e0-e4570361ec93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:09.735885483Z\",\n      \"endTime\": \"2026-09-28T05:23:14.561838226Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2s722b\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfeb60\u003e: \n    failed to create HCP cluster np-noedge-2s722b: failed waiting for cluster=\"np-noedge-2s722b\" in resourcegroup=\"rg-np-noedge-2s722b-ttwnzxstsj4p\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\",\n      \"name\": \"18b3752b-dfe7-413c-83e0-e4570361ec93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:09.735885483Z\",\n      \"endTime\": \"2026-09-28T05:23:14.561838226Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc65a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-6vnwh8vjnz8r\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\",\n      \"name\": \"0b3163de-a897-4f45-9b77-e785bdfab09f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:53.798033495Z\",\n      \"endTime\": \"2026-09-28T05:22:54.610455796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc65a0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-6vnwh8vjnz8r\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\",\n      \"name\": \"0b3163de-a897-4f45-9b77-e785bdfab09f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:53.798033495Z\",\n      \"endTime\": \"2026-09-28T05:22:54.610455796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"65926fb3ef21067b7449af3c8bad1b64cbed97824fcac64f72df97d6b9cae9eb","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"dc5aba529688a7c5d0344c6021726d3af95020d72a42d1fb5deae657451b585d","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb1198fd68f10a5126a38488adcab0e5698ddefc9bae064f4179ead558afbe1a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-8fvg54\" with version 4.22.15 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae0680\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-8fvg54: failed waiting for cluster=\"cluster-zstream-4-22-8fvg54\" in resourcegroup=\"rg-zstream-upgrade-4-22-8zfwm76xgnlj\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\",\n      \"name\": \"847235cf-8889-474c-8fc4-418de69191f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:18.057920805Z\",\n      \"endTime\": \"2026-09-28T19:39:25.918712924Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-8fvg54\" with version 4.22.15 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae0680\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-8fvg54: failed waiting for cluster=\"cluster-zstream-4-22-8fvg54\" in resourcegroup=\"rg-zstream-upgrade-4-22-8zfwm76xgnlj\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\",\n      \"name\": \"847235cf-8889-474c-8fc4-418de69191f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:18.057920805Z\",\n      \"endTime\": \"2026-09-28T19:39:25.918712924Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-27p526\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d20a60\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-27p526: failed waiting for cluster=\"cp-ystream-upgrade-4-22-27p526\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7drm72sqb8zr\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n      \"name\": \"3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:49.293974095Z\",\n      \"endTime\": \"2026-09-28T19:39:55.897408712Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-27p526\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d20a60\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-27p526: failed waiting for cluster=\"cp-ystream-upgrade-4-22-27p526\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7drm72sqb8zr\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n      \"name\": \"3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:49.293974095Z\",\n      \"endTime\": \"2026-09-28T19:39:55.897408712Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"883e6909996fc32433cd23f6eb465299bc9e4798451dc2382f9ae8bc9046e7f6","pr_number":7178,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"be02b062470985c866d3bce0cf574ecc16a3fffda367f6f061cf26ba857913fb","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce94a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j9cxlp\" in resourcegroup=\"rg-np-noedge-j9cxlp-9zvfnm962trj\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n      \"name\": \"3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T17:41:00.477362299Z\",\n      \"endTime\": \"2026-09-28T18:00:09.600349173Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce94a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j9cxlp\" in resourcegroup=\"rg-np-noedge-j9cxlp-9zvfnm962trj\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n      \"name\": \"3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T17:41:00.477362299Z\",\n      \"endTime\": \"2026-09-28T18:00:09.600349173Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f712c0\u003e: \n    failed to create NodePool np-wsvb2z: failed waiting for nodepool=\"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\" in resourcegroup=\"rg-candidate-4-20-slrjsf69bdm7\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\",\n      \"name\": \"119c8117-1a73-44d6-8ce8-38b6990193b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T17:43:51.079865069Z\",\n      \"endTime\": \"2026-09-28T18:02:59.523800413Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f712c0\u003e: \n    failed to create NodePool np-wsvb2z: failed waiting for nodepool=\"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\" in resourcegroup=\"rg-candidate-4-20-slrjsf69bdm7\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\",\n      \"name\": \"119c8117-1a73-44d6-8ce8-38b6990193b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T17:43:51.079865069Z\",\n      \"endTime\": \"2026-09-28T18:02:59.523800413Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"69e6cd1f2acc497163a0ca5d4d17a7b4e30d385be065f675b68acee156ec0d15","pr_number":7124,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"c178e32f3fda3138a135eb9850e356b70cd5c3cfe3691505d0aae5fb9b47aa1b","pr_number":7124,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[2,0,0,2,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-24T14:18:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00141d0c8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00141d0c8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e85578\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e85578\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"001bfebb218903b1b7737fc9b09415cf50b63b5c1c904d8a608f8a8de2c5cb07","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"e31b9c3e430438e0c3a30a2dbdd5700e039ffeab1240ef87b983e42b9cff5173","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-20T19:01:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","pr_number":7124,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","pr_number":7124,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-20T19:01:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7124,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7124,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb","failure_pattern":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T13:34:43Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":1}],"full_error_samples":["time=2026-09-23T14:16:23.495Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-23T14:17:09.991Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","occurred_at":"2026-09-23T13:34:43Z","signature_id":"15f55de3f2ea759d42d39cc08a545eae0c5c976acedd44ef9735b3b78e99695d","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"08ad18e89c33b55d4b3e771dd8d81006f379c5497ad8187cd5d751dd18c0ebc4","failure_pattern":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal: Timed out after \u003cduration\u003es.","search_query":"Timed out after 600.000s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:164 with:\nexpected HTTP 200 from sample app via internal LB, got 503\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:164 with:\nexpected HTTP 200 from sample app via internal LB, got 503\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"d77acb6f3606c90c5b93fb7307e047cf9fa16eec87a5e89c1c9e889994c91696","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","failure_pattern":"DataCurrent condition must be True","search_query":"DataCurrent condition must be True","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T12:46:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"a0937d42fb845e19eba0547c56962838f60fa2e7d16ff0ed1e5bc2f5dc2708a7","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"033a35a94b422f43f42fb3fd7d6be1a15babbad3631288f522120821e13fb3db","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-apiserver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-26r6nt\" with version 4.20.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106e580\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-26r6nt: failed waiting for cluster=\"cluster-zstream-4-20-26r6nt\" in resourcegroup=\"rg-zstream-upgrade-4-20-6bpd6fpsn6s4\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n      \"name\": \"4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:43.832400833Z\",\n      \"endTime\": \"2026-09-28T13:22:51.48967289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-apiserver, openshift-route-controller-manager, catalog-operator, community-operators-catalog, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-storage-operator, openshift-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, ingress-operator, cluster-policy-controller, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-26r6nt\" with version 4.20.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106e580\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-26r6nt: failed waiting for cluster=\"cluster-zstream-4-20-26r6nt\" in resourcegroup=\"rg-zstream-upgrade-4-20-6bpd6fpsn6s4\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n      \"name\": \"4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:43.832400833Z\",\n      \"endTime\": \"2026-09-28T13:22:51.48967289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-apiserver, openshift-route-controller-manager, catalog-operator, community-operators-catalog, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-storage-operator, openshift-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, ingress-operator, cluster-policy-controller, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"f7d62dbf65a929971d7b445f4669c5e565d9157cf38c9ff8ed6b211c77c0fe91","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0dd45aef10b8391c9e5053879220fb6ef98b320a9e44f86e49414d35d0aff7ee","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c41e0\u003e: \n    failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-jfwb2dgmj26t\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n      \"name\": \"a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:17.858913168Z\",\n      \"endTime\": \"2026-09-28T05:23:24.118546984Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c41e0\u003e: \n    failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-jfwb2dgmj26t\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n      \"name\": \"a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:17.858913168Z\",\n      \"endTime\": \"2026-09-28T05:23:24.118546984Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"f7515d3eb405f48e343776d3361bb29d0c59e96063451c01a874bff6974c8467","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"29d7e57f52a348cb16f8150b148a3d89306bfe96e3707f80643c1a84b22eed32","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, kube-apiserver, openshift-apiserver, packageserver, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010432a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-66ncz7vqwm2z\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\",\n      \"name\": \"c35d648d-395d-429f-9f2d-24eed4314d86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:11.81451702Z\",\n      \"endTime\": \"2026-09-28T19:39:16.076223121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010432a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-66ncz7vqwm2z\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\",\n      \"name\": \"c35d648d-395d-429f-9f2d-24eed4314d86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:11.81451702Z\",\n      \"endTime\": \"2026-09-28T19:39:16.076223121Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"8b5a7eabb5b08cdfee8da155099ec3786439ab3265705a61822bd89213b242a9","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bde990027a86d67f6f571e540c7c41a4153474968e2d1cf8aaae5f61d4176490","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6kj9nw5mjrc2/cluster-hs-pre-fvbpxs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db85e0\u003e: \n    failed to create HCP cluster cluster-hs-pre-fvbpxs: failed waiting for cluster=\"cluster-hs-pre-fvbpxs\" in resourcegroup=\"rg-hs-pre--6kj9nw5mjrc2\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\",\n      \"name\": \"2f78ab67-468f-449c-890b-25423489ffcd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:03.674200437Z\",\n      \"endTime\": \"2026-09-28T05:23:04.406727689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6kj9nw5mjrc2/cluster-hs-pre-fvbpxs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db85e0\u003e: \n    failed to create HCP cluster cluster-hs-pre-fvbpxs: failed waiting for cluster=\"cluster-hs-pre-fvbpxs\" in resourcegroup=\"rg-hs-pre--6kj9nw5mjrc2\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\",\n      \"name\": \"2f78ab67-468f-449c-890b-25423489ffcd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:03.674200437Z\",\n      \"endTime\": \"2026-09-28T05:23:04.406727689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"589125c66cd0fbbb7e8b00aa0c8a7cb91a2f86a1961306a34f41a9f722db5222","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b586f8fea86ba211104459fd89bb5ef2ea79917b36b639db22bd6e7bf973221c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae31a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5cbtfzkp497x\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n      \"name\": \"6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:33.784163345Z\",\n      \"endTime\": \"2026-09-28T05:22:34.354425424Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae31a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5cbtfzkp497x\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n      \"name\": \"6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:33.784163345Z\",\n      \"endTime\": \"2026-09-28T05:22:34.354425424Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"59d450dd267abc9e54a533d56b6c387e30b1b1a9dc2279ecaa1ac415c1887478","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f78894a0f745284e7b454f80e662337a874e257b6590d707b4a761a42e23b9f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-26T21:52:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b86200\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n      \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n      \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b86200\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-9464kjvgqp5t\" to finish creating: GET https://rp.j0737536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n      \"name\": \"34a5d5ce-cb5b-4d49-9d05-cf14dfa067ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:26:42.363515696Z\",\n      \"endTime\": \"2026-09-26T22:55:52.65701083Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, certified-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103965959260737536","occurred_at":"2026-09-26T21:52:29Z","signature_id":"214f1ab4117af1368e93bfb8d87ddb3947a25a6ea700ad87128366123ae0719e","pr_number":7185,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3b936310d5449729e7fddac902aac9547b4c9f95ec0132ddb43eddb70da02556","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb0840\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rbwvmrnzw8j5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\",\n      \"name\": \"5bdae3db-e612-4f27-8693-02200538e08e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:23.74994885Z\",\n      \"endTime\": \"2026-09-28T05:23:24.026384001Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb0840\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rbwvmrnzw8j5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\",\n      \"name\": \"5bdae3db-e612-4f27-8693-02200538e08e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:23.74994885Z\",\n      \"endTime\": \"2026-09-28T05:23:24.026384001Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"772c0837ffbd02ac36ebd76dd5378bc1ffed78fbbc057c3ba94ac4fd6bb2e9af","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106c320\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdqkz9rfmldx\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\",\n      \"name\": \"0c723202-68f5-41c7-a5d4-762157d31d23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:53.997312592Z\",\n      \"endTime\": \"2026-09-28T05:22:54.143428969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106c320\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdqkz9rfmldx\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\",\n      \"name\": \"0c723202-68f5-41c7-a5d4-762157d31d23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:53:53.997312592Z\",\n      \"endTime\": \"2026-09-28T05:22:54.143428969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"96243ae66c6d39c7d59fea442c69e34bfc8d95860eb89e28f5e0f3d2d01c561b","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8f36771e9fe45b5eb7816e2eac12b45993b1ce25d5e99408f0060a841a7ead5d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gng5crkc9wh8/cluster-candidate-4-22-w8wr72 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00117a200\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-w8wr72: failed waiting for cluster=\"cluster-candidate-4-22-w8wr72\" in resourcegroup=\"rg-candidate-4-22-gng5crkc9wh8\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n      \"name\": \"00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:03.970731237Z\",\n      \"endTime\": \"2026-09-28T05:23:04.188577598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, router; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gng5crkc9wh8/cluster-candidate-4-22-w8wr72 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00117a200\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-w8wr72: failed waiting for cluster=\"cluster-candidate-4-22-w8wr72\" in resourcegroup=\"rg-candidate-4-22-gng5crkc9wh8\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n      \"name\": \"00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:54:03.970731237Z\",\n      \"endTime\": \"2026-09-28T05:23:04.188577598Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, router; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"af9b004a3ce14cb7f4279f4399fb02c6d5dcd5d659994a984db66fa7d804f482","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c073278b4df6afb8cf91007966cde1fa7c3453fd0ebf082892677572d0b03f07","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint \u003chcp-api-host\u003e failed: Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-rxbdcb\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090b40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-rxbdcb: failed waiting for cluster=\"cp-ystream-upgrade-4-23-rxbdcb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-jvcp8dmnbrt7\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n      \"name\": \"1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T07:17:56.829077997Z\",\n      \"endTime\": \"2026-09-28T07:47:03.158495671Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud failed: Get \\\"https://api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud:443/healthz?verbose\\\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-rxbdcb\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090b40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-rxbdcb: failed waiting for cluster=\"cp-ystream-upgrade-4-23-rxbdcb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-jvcp8dmnbrt7\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n      \"name\": \"1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T07:17:56.829077997Z\",\n      \"endTime\": \"2026-09-28T07:47:03.158495671Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud failed: Get \\\"https://api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud:443/healthz?verbose\\\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","signature_id":"8f8369e8e139f64a46a3b6dd8d438f715d6bab665ea52bac944061720f9f66e7","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f63c45d9c3e4d65908b630f98974bc45ae47fd02522293aa9b16d6f8e854d56b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f2c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p82fzl2z5vk5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\",\n      \"name\": \"03f41a1e-bd33-4df1-9f47-56824bb74022\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:55:14.362679822Z\",\n      \"endTime\": \"2026-09-28T05:24:15.510395217Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f2c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p82fzl2z5vk5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\",\n      \"name\": \"03f41a1e-bd33-4df1-9f47-56824bb74022\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T04:55:14.362679822Z\",\n      \"endTime\": \"2026-09-28T05:24:15.510395217Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","signature_id":"a9535031eecea83204e2a98f841c802977ff506d08a46a66165d236a2a58cbfc","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"10442c188c833ffedc7ef4da292665baff0c37e673939cc22c4ae73341b8897f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, cluster-image-registry-operator, community-operators-catalog, hosted-cluster-config-operator, kube-apiserver, openshift-apiserver, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-c89c6r\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10480\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-c89c6r: failed waiting for cluster=\"cluster-zstream-5-0-c89c6r\" in resourcegroup=\"rg-zstream-upgrade-5-0-6kc9p4vcvjqq\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n      \"name\": \"37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:45.206355596Z\",\n      \"endTime\": \"2026-09-28T13:22:51.172490782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 26m23s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-c89c6r\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10480\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-c89c6r: failed waiting for cluster=\"cluster-zstream-5-0-c89c6r\" in resourcegroup=\"rg-zstream-upgrade-5-0-6kc9p4vcvjqq\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n      \"name\": \"37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T12:53:45.206355596Z\",\n      \"endTime\": \"2026-09-28T13:22:51.172490782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 26m23s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"4f4e5752545f8f46e297ca73e9f4e1a12d4ee1ba77d68cd29f8525db53cf046a","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d71a4287cac7c4db8dd3797eb946a1bbbe45de0ccd7e568821596e513bc09abb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, kube-apiserver, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf61a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lhz84lf5nw65\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\",\n      \"name\": \"045ddd50-d43d-42a0-bcda-90888c378119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:11:03.945176993Z\",\n      \"endTime\": \"2026-09-28T19:40:05.721943753Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m29s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf61a0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lhz84lf5nw65\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\",\n      \"name\": \"045ddd50-d43d-42a0-bcda-90888c378119\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:11:03.945176993Z\",\n      \"endTime\": \"2026-09-28T19:40:05.721943753Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m29s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"0b5f642a5234bd0a643829d4d6a02a3ececb750e0fc106623478f6d9c53767e1","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c7fe0af289e2969c7eb39488d64d294fadea2123920704d81e8b66750b4f367b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, kube-controller-manager, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6mht66l8hhvp/cluster-candidate-4-22-hxvn25 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00070f360\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-hxvn25: failed waiting for cluster=\"cluster-candidate-4-22-hxvn25\" in resourcegroup=\"rg-candidate-4-22-6mht66l8hhvp\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n      \"name\": \"889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:32.906905274Z\",\n      \"endTime\": \"2026-09-28T19:39:35.528660951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6mht66l8hhvp/cluster-candidate-4-22-hxvn25 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00070f360\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-hxvn25: failed waiting for cluster=\"cluster-candidate-4-22-hxvn25\" in resourcegroup=\"rg-candidate-4-22-6mht66l8hhvp\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n      \"name\": \"889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:32.906905274Z\",\n      \"endTime\": \"2026-09-28T19:39:35.528660951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"0e99260da9e615074014e3173244136846aafa91e1232cd4f863676446c4f26c","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e3e3516e4f9f9dcb22ed9c70498bb550fb681b3348ceed9a50619cee4e93ae7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060c440\u003e: \n    failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-br7vgttbd9zx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n      \"name\": \"15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:44.986989982Z\",\n      \"endTime\": \"2026-09-28T19:39:45.910063992Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 24m39s ago; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060c440\u003e: \n    failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-br7vgttbd9zx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n      \"name\": \"15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:44.986989982Z\",\n      \"endTime\": \"2026-09-28T19:39:45.910063992Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 24m39s ago; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"65f361db2ca8a207b71ef63dfd951eb9adaf0933223cf15a0e8cb91c8b77bb7d","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T16:19:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4e300\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k7vshcl5nh9s\" to finish creating: GET https://rp.j8937088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n      \"name\": \"60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T17:21:27.919326113Z\",\n      \"endTime\": \"2026-09-25T17:50:31.48468713Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 35s ago; version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 5m19s ago; version 4.21.35 is Partial (want Completed), started 6m43s ago; version 4.21.35 is Partial (want Completed), started 8m41s ago; version 4.21.35 is Partial (want Completed), started 12m14s ago; version 4.21.35 is Partial (want Completed), started 13m25s ago; version 4.21.35 is Partial (want Completed), started 13m48s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4e300\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-k7vshcl5nh9s\" to finish creating: GET https://rp.j8937088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n      \"name\": \"60dc2458-30c4-43c0-9c09-17e7a502b57d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T17:21:27.919326113Z\",\n      \"endTime\": \"2026-09-25T17:50:31.48468713Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 35s ago; version 4.21.35 is Partial (want Completed), started 3m12s ago; version 4.21.35 is Partial (want Completed), started 5m19s ago; version 4.21.35 is Partial (want Completed), started 6m43s ago; version 4.21.35 is Partial (want Completed), started 8m41s ago; version 4.21.35 is Partial (want Completed), started 12m14s ago; version 4.21.35 is Partial (want Completed), started 13m25s ago; version 4.21.35 is Partial (want Completed), started 13m48s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103519715358937088","occurred_at":"2026-09-25T16:19:16Z","signature_id":"80d23cd3f5a37fa8cc3ba788b04667c12b2e9abb864a0b69c2b97bc1936b5e91","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ad05e6869aca7a4924e8d188ef3355a20d9fcfa6fdf28bdb0dceaf7eb87d2816","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4rl6n98blqls/cluster-candidate-5-0-9zqn4p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22920\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9zqn4p: failed waiting for cluster=\"cluster-candidate-5-0-9zqn4p\" in resourcegroup=\"rg-candidate-5-0-4rl6n98blqls\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\",\n      \"name\": \"b919a89b-ddbd-409e-ae0b-825760357927\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:07.120894243Z\",\n      \"endTime\": \"2026-09-28T19:39:16.873419842Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4rl6n98blqls/cluster-candidate-5-0-9zqn4p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22920\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-9zqn4p: failed waiting for cluster=\"cluster-candidate-5-0-9zqn4p\" in resourcegroup=\"rg-candidate-5-0-4rl6n98blqls\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\",\n      \"name\": \"b919a89b-ddbd-409e-ae0b-825760357927\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T19:10:07.120894243Z\",\n      \"endTime\": \"2026-09-28T19:39:16.873419842Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"1aeed5a6216935410be4afa173f9b01087ed1978311df09a7db6cc003d86fa29","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T20:01:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"35009a2d8ec15af6629b3b447ac330d1c5376735efbdae49f03535446218b76a","pr_number":7110,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb9a7ba2d967015ac741b1ba2401e95d8184541adbccfc98565936e10e7aa68c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity; detail message Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T04:31:30Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-22T04:51:02.686Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-22T04:51:04.267Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-22T04:51:06.089Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9675136-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985\ntime=2026-09-22T05:04:09.339Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9675136-svc/providers/Microsoft.Resources/deployments/96622666cca941a93b9a8651dce6827ae1f27fb7972b41f9caa78871db9bc985/operationStatuses/08584115542204326289\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9675136-svc/providers/Microsoft.Resources/deployments/cluster-olgmgzuso436u\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9675136-svc/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6516/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102254439539675136","occurred_at":"2026-09-22T04:31:30Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6516,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e5f4977400f08751c42a094e098f25015a196161edfd5a441bd871ef2846ba3e","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T11:56:22Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-22T12:18:17.689Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T12:45:29.994Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T12:18:17.695906444Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-22T12:18:20.00208003Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T12:18:20.14445826Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992055155Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"CF0FD6F1065F90B5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT8.5695733S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 44 for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992133756Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DD6EAF8731EA851B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M23.5357859S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992147237Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9E2ABCCAB49D4C81\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M23.3548981S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992157287Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"42D7531EF6508071\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:43:18.8605679Z\\\",\\\"duration\\\":\\\"PT1M53.0336442S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992166697Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1B516DD7916AE405\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:18:26.1875166Z\\\",\\\"duration\\\":\\\"PT26M47.0458862S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992178797Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2569996FCD48888B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-22T12:18:26.1875166Z\\\",\\\"duration\\\":\\\"PT24M52.6018784S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T12:45:29.992201978Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2830976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j2830976-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115273851533055\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2830976-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j2830976-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 44 for family StandardDdsv6Family for region centralus. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"27m12.296304024s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102366391112830976","occurred_at":"2026-09-22T11:56:22Z","signature_id":"4e96560f2ff6d93bac0f0e5cfb31586c4f9dc831e4a29778f4268da2aebff893","pr_number":6590,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56fd2feac0bf2fd733880f128445ced8b465f2cded0a3ea3fc3aa25b8699b45c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Failed; provider Microsoft.Kusto","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kusto-grant-ingest (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T04:16:59Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/kusto-grant-ingest (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-27T04:55:22.142Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 description=\"Step kusto-grant-ingest\\n  Kind: ARM\\n  Template: templates/kusto-grant-ingest.bicep\\n  Parameters: configurations/kusto-mgmt-ingest.tmpl.bicepparam\"\ntime=2026-09-27T04:55:22.662Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2\ntime=2026-09-27T04:55:23.741Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 deployment=2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8656896-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80\ntime=2026-09-27T04:59:54.839Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=kusto-grant-ingest stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8656896-mgmt-2/providers/Microsoft.Resources/deployments/2e905e8738711f49918794c12ee7bdb55d7d9202820d0f6abb23829107a4ce80/operationStatuses/08584111219623969246\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/grantKusto-hcp-ja2yqt7pclopi\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-kusto-us/providers/Microsoft.Kusto/clusters/hcp-dev-us-2/databases/HostedControlPlaneLogs/principalAssignments/grant-b6fd691c-fba0-50f9-bb53-13af017771ec\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Internal Server Error\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104062724328656896","occurred_at":"2026-09-27T04:16:59Z","signature_id":"24248bfb9172326e331f0bc70fbaea3432445ca771da2ecd7e9aa55cd2f3ee9a","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T16:41:24Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T17:25:13.342Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-09-24T17:25:13.641Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-09-24T17:25:14.167Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4687104-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3\ntime=2026-09-24T17:25:44.283Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3/operationStatuses/08584113361715956682\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/profile-bmfsiknuehg4q\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162898384687104","occurred_at":"2026-09-24T16:41:24Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":20,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T02:06:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98ae0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cordon-hcp-cluster-rck8ht\" in resourcegroup=\"rg-cordon-test-tlds99sfrm5j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cordon-test-tlds99sfrm5j/providers/Microsoft.Resources/deployments/customer-infra-cordon-hcp-cluster-rck8ht/operationStatuses/08584113017077419903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b98ae0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cordon-hcp-cluster-rck8ht\" in resourcegroup=\"rg-cordon-test-tlds99sfrm5j\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cordon-test-tlds99sfrm5j/providers/Microsoft.Resources/deployments/customer-infra-cordon-hcp-cluster-rck8ht/operationStatuses/08584113017077419903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"InternalServerError\\\",\\r\\n        \\\"message\\\": \\\"An error occurred.\\\",\\r\\n        \\\"details\\\": []\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103305015505457152","occurred_at":"2026-09-25T02:06:07Z","signature_id":"4e82558e3f130c0b3cf620c520d17c6d0e6517b368b13c167cb42967e19e28ba","pr_number":7141,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4adf443619cbd9518f5388bac91db3b79c0430d7fca6e294ba38f40513bd0e3a","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca2e60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-z4h7jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-z4h7jf/operationStatuses/08584110018702456355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name/providers/Microsoft.Authorization/roleAssignments/ce4c794c-50e9-565e-99cc-b7dd7e08c5a9' at line '1' and column '29054'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 29054,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/e5697265-1ef6-54a3-8e48-b96178ae0611' at line '1' and column '30683'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 30683,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca2e60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-z4h7jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-z4h7jf/operationStatuses/08584110018702456355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name/providers/Microsoft.Authorization/roleAssignments/ce4c794c-50e9-565e-99cc-b7dd7e08c5a9' at line '1' and column '29054'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 29054,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n            \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/e5697265-1ef6-54a3-8e48-b96178ae0611' at line '1' and column '30683'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n            \\\"additionalInfo\\\": [\\r\\n              {\\r\\n                \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n                \\\"info\\\": {\\r\\n                  \\\"lineNumber\\\": 1,\\r\\n                  \\\"linePosition\\\": 30683,\\r\\n                  \\\"path\\\": \\\"\\\"\\r\\n                }\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104562485683032064","occurred_at":"2026-09-28T13:22:52Z","signature_id":"8e7656a6acdc1e558b5e327917ebbb4231a9f78559a33ae638709b57cfbe1f81","pr_number":7104,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"974128b582e7e400d6215f2a944dca5aafec67820d9ccf78df791931926850f4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ReconcileVMSSAgentPoolFailed; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T00:02:27Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-22T00:28:42.028Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T00:31:19.046Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T00:28:42.035099926Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-22T00:28:44.199184892Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T00:28:44.294775263Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-22T00:31:19.041004649Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B5981CAA76057630\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5677184-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-22T00:28:49.2033491Z\\\",\\\"duration\\\":\\\"PT2M28.4887595S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ReconcileVMSSAgentPoolFailed\\\",\\\"message\\\":\\\"Internal server error\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-22T00:31:19.041107186Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5677184-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5677184-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584115699609930281\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5677184-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5677184-mgmt-1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n        \\\\\\\"details\\\\\\\": [\\\\n          {\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ReconcileVMSSAgentPoolFailed\\\\\\\",\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Internal server error\\\\\\\"\\\\n          }\\\\n        ]\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"2m37.00601594s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102186730835677184","occurred_at":"2026-09-22T00:02:27Z","signature_id":"a25c2fb7b5fe7b3b703abe9fe9cdab41f229d7afd7a57f710d0cdf443b52a3a0","pr_number":5846,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T11:48:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c324a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-rjv7q9\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9crmxlzr24l9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-9crmxlzr24l9/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-rjv7q9/operationStatuses/08584112672274846775\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-xafodxhow6npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c324a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-rjv7q9\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-9crmxlzr24l9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-9crmxlzr24l9/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-rjv7q9/operationStatuses/08584112672274846775\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-xafodxhow6npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103451563560275968","occurred_at":"2026-09-25T11:48:27Z","signature_id":"12ae308cd1789be846262f9aa7e8cd0ffc37068656f3e5bc64966bcafd3a0dec","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506","failure_pattern":"ERROR CODE: InternalServerError; context failed to retrieve serial console logs for VM \"\u003cvm\u003e\"","search_query":"\"message\": \"Internal server error.\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T16:00:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","occurred_at":"2026-09-23T16:00:38Z","signature_id":"2074bff381a20809ab3d0b56df62ec1c9c4cecad07bc3b02a8f9d837a90a1b6a","pr_number":7112,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6ddfc035a80bbbf2ad88605092a69732d98a1147956895ca4acb58224e5834fe","failure_pattern":"ERROR CODE: InternalServerError; detail code OperationNotAllowed; detail message Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-\u003cid\u003e' since another operation is in progress.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T12:26:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00085dbd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-566qcntr7gdn\" to finish creating: GET https://rp.j4621824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\",\n      \"name\": \"47438056-285b-41f2-ab31-9d3620f43054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:28:05.551349476Z\",\n      \"endTime\": \"2026-09-25T13:47:09.734640501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-h9wwd-6cst6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-h9wwd-6cst6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-h9wwd-6cst6' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00085dbd8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-566qcntr7gdn\" to finish creating: GET https://rp.j4621824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47438056-285b-41f2-ab31-9d3620f43054\",\n      \"name\": \"47438056-285b-41f2-ab31-9d3620f43054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T13:28:05.551349476Z\",\n      \"endTime\": \"2026-09-25T13:47:09.734640501Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-h9wwd-6cst6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-h9wwd-6cst6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-h9wwd-6cst6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-566qcntr7gdn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-h9wwd-6cst6/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-h9wwd-6cst6' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7163/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103461242604621824","occurred_at":"2026-09-25T12:26:55Z","signature_id":"b45947755f19be26b32e2accffc4cc0be60dad387febd577240f60063bf46a38","pr_number":7163,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c62a41b1ffa48cd7eb5519e7f4bdc548f260009543ddaf09220fa0719901910a","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceede... message truncated; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T18:36:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n      \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n      \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"43e7b49a53b9f05fe2662927a58e42b643a0bf4af76dedddd2bff94bc094d954","pr_number":7141,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3d6035dc15a6de29702ad772c845059ee997267e8ce8729729aed4e2cee290dd","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional token... message truncated; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T02:58:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006377a0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8lt5rxb4tbjz\" to finish creating: GET https://rp.j7857664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n      \"name\": \"ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T04:05:25.290345016Z\",\n      \"endTime\": \"2026-09-25T04:24:25.319865016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-6qhsv-d96qd: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-8lt5rxb4tbjz--managed/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8lt5rxb4tbjz--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006377a0\u003e: \n    failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8lt5rxb4tbjz\" to finish creating: GET https://rp.j7857664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n      \"name\": \"ed95f0cc-a570-4ff6-b5aa-553b0f2e32b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T04:05:25.290345016Z\",\n      \"endTime\": \"2026-09-25T04:24:25.319865016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-6qhsv-d96qd: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-8lt5rxb4tbjz--managed/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8lt5rxb4tbjz--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-6qhsv-d96qd-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318277487857664","occurred_at":"2026-09-25T02:58:49Z","signature_id":"ba0a7ce2e48b37c08aaa14a387af90804db2c47729b8040cba4bb165d499132f","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"372d0c16d6516fbc2516124c1f8eb533277ffb462595208eacf857f749c90520","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T21:23:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145fb18\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n      \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145fb18\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n      \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"142775d0e7b84904771b9c9baeca0e991ed25c62693ee1753f67f0e8192808e1","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-16T10:05:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000687000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdlktwxnm8vh\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\",\n      \"name\": \"9fb52b81-388d-4086-8055-43b274743b11\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:41:09.989503835Z\",\n      \"endTime\": \"2026-09-28T23:41:15.954914297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000687000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdlktwxnm8vh\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\",\n      \"name\": \"9fb52b81-388d-4086-8055-43b274743b11\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:41:09.989503835Z\",\n      \"endTime\": \"2026-09-28T23:41:15.954914297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","signature_id":"e75e1ad4b0432801e2acb69e7608842fb32bc692d9078a7d6cfc759e2f16fd9a","pr_number":6922,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0e33bf9c876122b5f27225338e1ee7dd193ea167c268da2c880fb16fb083cfb4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T11:47:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n      \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n      \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","signature_id":"a1699dd27afd8a16f801afb5b59631643f3ad5b14b803812cf5d901f58fe2d87","pr_number":7123,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, dns-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T19:19:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","signature_id":"3fec0289f4e859b2672b4ecfd1c53de11f801ca25e568cb6fd49601565fabc04","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b4593ef0dd9fbe369bd9dc5fb00d11d789b0f9fff11fea0544e9abdf8738d8b1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T19:22:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b80880\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n      \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b80880\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n      \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"073b0db0ceea3f28f404fe1b7904861cb57586c11302c87c1ccfd472e1f2a7a3","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64bc5ac277db781b54ff950d9ea3c0c424d37517be1bdac9b31411f402f5258f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T22:38:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n    failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n      \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n      \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n    failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n      \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n      \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","signature_id":"1074b07740a45107502c1b1a3cd307b20f68100d4367f76d22ba40ec484f0d3d","pr_number":7123,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"faaf4b1f5915530cc590a6d965e51a07e2893bcc928bcf257ec4329e7f00397f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T18:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002703dd0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n      \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002703dd0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n      \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103182792887111680","occurred_at":"2026-09-24T18:00:08Z","signature_id":"5bf349cb8ec9f251b4fddcd8eba4f881e8bb266c023ecc7561970e4ac0e37274","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b6695a9c8ed566b40df73975745c0c7f92e4f64ce36419ead0b585fc65511cae","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T11:30:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00134acc0\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-nfdpj-h6dps]\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-z8btrh6l7hvp\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\",\n      \"name\": \"9dfb13f2-af5d-471c-b272-b1c35505253f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T12:51:15.748310239Z\",\n      \"endTime\": \"2026-09-25T13:10:18.43513357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00134acc0\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-nfdpj-h6dps]\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-z8btrh6l7hvp\" to finish creating: GET https://rp.j2523264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9dfb13f2-af5d-471c-b272-b1c35505253f\",\n      \"name\": \"9dfb13f2-af5d-471c-b272-b1c35505253f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T12:51:15.748310239Z\",\n      \"endTime\": \"2026-09-25T13:10:18.43513357Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-nfdpj-h6dps: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-z8btrh6l7hvp--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-nfdpj-h6dps to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103447011792523264","occurred_at":"2026-09-25T11:30:21Z","signature_id":"3d00756f058ac76508c5214768bdd5e72a8833c6eb5c3c560deb2bc53bc00897","pr_number":7165,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"881c8a92b161acdd9b8088fd7670ee486a4176a53d8400a1bad38528667ae5f4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T19:22:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n      \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n      \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"b24161097af08c2ace32acbd8380e530f701f116d3593a7a6af06880e3f2dbd7","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-16T15:55:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ca85b8\u003e: \n    resource group \"admin-api-serialconsole-st5xdz2xcc6m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-st5xdz2xcc6m\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n      \"name\": \"3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T01:06:48.076247478Z\",\n      \"endTime\": \"2026-09-28T01:30:48.731406356Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmeectlf6lq0ip66b1ckuumq6c71t still exists (deletion dispatched at 2026-09-28T01:06:49Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ca85b8\u003e: \n    resource group \"admin-api-serialconsole-st5xdz2xcc6m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-st5xdz2xcc6m\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n      \"name\": \"3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T01:06:48.076247478Z\",\n      \"endTime\": \"2026-09-28T01:30:48.731406356Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmeectlf6lq0ip66b1ckuumq6c71t still exists (deletion dispatched at 2026-09-28T01:06:49Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","signature_id":"2e6206962622fa23645da22c99b31d5211fa88fc0db515931249a87ce69c3e13","pr_number":7041,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f04078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xvhgrrnh7hjx\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n      \"name\": \"177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T23:54:44.577772615Z\",\n      \"endTime\": \"2026-09-23T00:18:50.338452518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001f04078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-xvhgrrnh7hjx\" to finish deleting: GET https://rp.j2688256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n      \"name\": \"177d426f-dd32-4799-8b9b-eb1aaa182ca9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T23:54:44.577772615Z\",\n      \"endTime\": \"2026-09-23T00:18:50.338452518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","signature_id":"acc491f73905825ad00c1e614ceac5874bb38c51bbce62f68329aa496898fecb","pr_number":6359,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db15661ea057fcdb86696aa934e0ebb9321ec309068c024420faf52e725ffb06","failure_pattern":"ERROR: Please run 'az login' to setup account.","search_query":"ERROR: Please run 'az login' to setup account.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T09:08:54Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-25T09:33:51.597Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n  Kind: Shell\\n  Command: ./swift-vnet.sh\\n\"\ntime=2026-09-25T09:33:52.825Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\\nERROR: Please run 'az login' to setup account.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103411413878837248","occurred_at":"2026-09-25T09:08:54Z","signature_id":"21658b9b3d6cf9bf31715ba32c333953291d0d0fe4a6e4e41252f24603e05dff","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f9989e753db1daf9717992966720fb7c20aebb56a0a3e0abe95adf30bb4255a7","failure_pattern":"Process did not finish before 15m0s timeout","search_query":"Process did not finish before 15m0s timeout","failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"unknown","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:178\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 15m0s timeout\",\"severity\":\"error\",\"time\":\"2026-09-23T00:52:30Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","signature_id":"20a552c9ef65c7031e4db49271edb6fa7f672d4bde424eea2c61b746d13edef7","pr_number":6359,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002725230\u003e: \n    failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002725230\u003e: \n    failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"19c4e7bd3a878120f8a1e6eec2558373cd830b2818377cfd788cbadd533fad7a","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-21T21:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:299]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00123cdb0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:299]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00123cdb0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","signature_id":"d992f3cd9ef02517f10c183a1d637491540983decdd93733599aca19cf494b86","pr_number":6900,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b2ed02918fbe605f5650b4b12180f78e9cff28da794fad98a9e5cfcbe1ec8f13","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb0020\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fb0020\u003e: \n    VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","signature_id":"af77cab624dfbc834f82810d101f0f6241da610632987ef8d562e3944af01ee4","pr_number":7124,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"278b745a86bd068f5605c35799bb8d30b899c82f446eb91deb2f0cab51e1e047","failure_pattern":"VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","search_query":"VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T18:09:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:334]: failed to pull image from private ACR acrpullcreatebgsbfthf8tqk.azurecr.io — the credential provider did not authenticate the pull\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001270ec0\u003e: \n    VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n      - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 08125a7d-2aeb-4cda-8888-4d2636d60fd2; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 48c612c6-0cab-4944-9d49-fbdaec01dcc2\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:334]: failed to pull image from private ACR acrpullcreatebgsbfthf8tqk.azurecr.io — the credential provider did not authenticate the pull\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001270ec0\u003e: \n    VerifyImagePulled(acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal) timed out after 3m0.007s: expected image from acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n      - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatebgsbfthf8tqk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 08125a7d-2aeb-4cda-8888-4d2636d60fd2; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 48c612c6-0cab-4944-9d49-fbdaec01dcc2\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103547336255868928","occurred_at":"2026-09-25T18:09:01Z","signature_id":"5673b7d1af626ca3af29742bda73744e5ecefedca976779ee4bbf1bbdd725701","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7128002351c39d7ffe6bc9e97f3e7bea8a864e7268f53656aca162804a977c0a","failure_pattern":"VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","search_query":"VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T20:17:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:397]: failed to pull image after updating containerRegistry to second MI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92200\u003e: \n    VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n      - pod acr-pull-test-after-update container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: eefe0445-30e7-478a-8283-14278233fefa; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c9d8d031-808a-4e35-9c9c-bff18e860fbb\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:397]: failed to pull image after updating containerRegistry to second MI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92200\u003e: \n    VerifyImagePulled(acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal) timed out after 3m0.006s: expected image from acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal in namespace acr-pull-test: image pull errors:\n      - pod acr-pull-test-after-update container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullcreatefddp5k2l7rsk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: eefe0445-30e7-478a-8283-14278233fefa; artifact err: get manifest: build image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c9d8d031-808a-4e35-9c9c-bff18e860fbb\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103579630278545408","occurred_at":"2026-09-25T20:17:21Z","signature_id":"04e92aed41dc4397943b4c76c401adb73bdb624b4bd11916358f9767c8bfe9d6","pr_number":7169,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"81045ed261487eb0846f85a3e15e6408c3110ed61e79109821b9fdbaebb04f4d","failure_pattern":"VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:","search_query":"VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullnsjbs8pwqtmk.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101eb60\u003e: \n    VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:\n      - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: e8a1ddc1-f3e1-4c99-ae14-bbe272e0b702; artifact err: image reference: get manifest from ref: create image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 7d311f24-e917-4750-a06e-94b359669923\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullnsjbs8pwqtmk.azurecr.io\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101eb60\u003e: \n    VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:\n      - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: e8a1ddc1-f3e1-4c99-ae14-bbe272e0b702; artifact err: image reference: get manifest from ref: create image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 7d311f24-e917-4750-a06e-94b359669923\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104634571122282496","occurred_at":"2026-09-28T18:08:52Z","signature_id":"4f3c97c58357c28ebf837088dc673173d0f261c2fec843f5f7f22dbbe9566892","pr_number":7181,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ea683689d213a80fd9693db9b372fae798477a56e7e530874ac1f5a4eddf2fd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080dc20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080dc20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n      - pod pull-secret-test container pull-test: ContainerCreating ()\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605132682432512","occurred_at":"2026-09-28T16:12:19Z","signature_id":"1cca4c55cd8817a60cc8a873fae0fecf46a8fe0f006a5fe9f16d6906ad65f148","pr_number":7020,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001664240\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001664240\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104701088555864064","occurred_at":"2026-09-28T22:33:37Z","signature_id":"4eb025693d04a56b1a1cdecd99f4ccf2153a73ec54e9bcd017b4a02505bf1b0f","pr_number":7231,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0f6b13b4a76d7562287bf1face4f5d9fb1b45d4ebf0047290b8a707a05354ee","failure_pattern":"VerifyNodesReady failed: there are not ready nodes: [\u003cnode\u003e]","search_query":"VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T16:14:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014697a0\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014697a0\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-5bd8h-kb8d6]\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103518634033811456","occurred_at":"2026-09-25T16:14:55Z","signature_id":"206eded0e06122e841558aff57990edd60f07f91dcefa176fe7ded75bb991a39","pr_number":7053,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-25T14:02:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-xz45t.apps.aro.x9e6g8g9i5v5d8w.8r9s.j1359744.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103485306291359744","occurred_at":"2026-09-25T14:02:32Z","signature_id":"f069a5566b22b52075f385f50e2f83deea5b73d137ebc80a3f1b3a8567055d32","pr_number":7165,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27efdba0688c010a9407237cceb9d29c447c7f9bd7965cf5f46a9ce24313bf72","failure_pattern":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","search_query":"alert [hcp] userJourneySwiftRouterAvailability1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-22T22:46:36Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneySwiftRouterAvailability1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-22T23:53:49Z\n  Ended:       2026-09-23T00:03:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneySwiftRouterAvailability1h5m\", burn_rate_tier=\"fast\", cluster=\"ci01-j2688256-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2688256/providers/microsoft.monitor/accounts/hcps-j2688256\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"swift-router-availability\"\n  Description: SWIFT router-capacity availability is consuming its error budget at the fast burn rate."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102530027982688256","occurred_at":"2026-09-22T22:46:36Z","signature_id":"701a40a807847341bdf087ce02e65a9ab90dca8be0fe9f9ce4f46071198313de","pr_number":6359,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-24T14:18:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T15:19:32Z\n  Ended:       2026-09-24T16:14:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2833536-svc\", component=\"backend\", name=\"controlplaneversionstatuscollector\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue controlplaneversionstatuscollector has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"42836fa06f25aa02442410f2c7556dd0019e006d1c588c77355505f2e0955532","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T17:37:33Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-27T18:49:46Z\n  Ended:       2026-09-27T18:56:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j8175232-svc\", component=\"fleet\", name=\"nodepoolcontroller\", region=\"westus3\", severity=\"warning\"\n  Description: Fleet controller workqueue nodepoolcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104264194718175232","occurred_at":"2026-09-27T17:37:33Z","signature_id":"b1312384d9b2a53ea723368157d1cd9705e0e9f23816fc33749b5b53bb2fe8fb","pr_number":7157,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-24T06:52:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:49:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n  Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:50:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n  Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"dd3366bd92c2564cb6f449817e8c4d513e034b81a60fe1b01c561f9570ac3115","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-28T13:21:03Z\n  Ended:       2026-09-28T13:27:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"prometheus\", node=\"aks-userswft3-30800232-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft3-30800232-vmss000000 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","signature_id":"7e5bb140f45412694ed52c011bce64b635715a19e67b5f190f74b9ede9baf738","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-21T22:11:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T14:36:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j1676288-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1676288/providers/microsoft.monitor/accounts/services-j1676288\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c86cd5988-85nd2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j1676288-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-28T14:36:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j1676288-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1676288/providers/microsoft.monitor/accounts/services-j1676288\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c86cd5988-85nd2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j1676288-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104548669041676288","occurred_at":"2026-09-28T12:27:56Z","signature_id":"d2d57b385cfb20f234fa1e1737740b5d90198f87bf051ebb2a00dd1115909d3b","pr_number":7169,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d95ad1f73f14bcd5a00d745ec172e29456c0e319d8668700b8e6e2c26009682d","failure_pattern":"alert [svc] MaestroEventStuckUnreconciled fired","search_query":"alert [svc] MaestroEventStuckUnreconciled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroEventStuckUnreconciled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T15:23:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroEventStuckUnreconciled does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T16:32:06Z\n  Ended:       2026-09-25T16:39:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroEventStuckUnreconciled\", component=\"maestro\", region=\"westus3\", severity=\"warning\"\n  Description: A Maestro event has been unreconciled for 6m 6s. This likely indicates a lost Postgres NOTIFY message. The event will not be processed until the periodic sync runs (default 10 hours)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","signature_id":"080e2def46312d4b62996fecf6131bceb6d060ae87b921205929ef11720accf9","pr_number":7176,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5121c689bbab23ebd70308f98015ffe29584447a2e07ce08bd8d406a3c32c40f","failure_pattern":"alert [svc] MaestroSpecControllerReconcileErrors fired","search_query":"alert [svc] MaestroSpecControllerReconcileErrors fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroSpecControllerReconcileErrors does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T15:23:24Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroSpecControllerReconcileErrors does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T16:36:07Z\n  Ended:       2026-09-25T16:58:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroSpecControllerReconcileErrors\", component=\"maestro\", region=\"westus3\", severity=\"warning\"\n  Description: Maestro spec controller reconcile error rate is above 10% for the last 10 minutes. Resources may not be reaching management clusters. Current value: 20.38%."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103505655406006272","occurred_at":"2026-09-25T15:23:24Z","signature_id":"9c3b815afa64d94bbcc8e738bb6c0fb9a74eada3ce4bf0147c94d3c439b9f138","pr_number":7176,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-22T16:02:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.007s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.007s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102428363070115840","occurred_at":"2026-09-22T16:02:36Z","signature_id":"e3177ca20277054a46f71328a257ac164787c1802403c4af7bcab8b46b4dbf8d","pr_number":6503,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-23T23:26:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","occurred_at":"2026-09-23T23:26:51Z","signature_id":"c8ce66f37f9d13dc657060400db223a90301ac4438226c58a0c01cac44d1ec13","pr_number":5562,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dbba05f95aaac62366d47b6f23fc0ba2f14aa49eae9bd82309cadecd8332be34","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j6380032-mgmt-2\",\"clusterName\":\"ci01-j638003","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T18:40:27Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T19:21:37.035Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-24T19:51:36.670Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","signature_id":"3d35abb788cea68e7144d2ec2e84c98ce2744861ebc8486e8349a5f051f0df62","pr_number":7045,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6469a04d9641eaa98fbab1139ee6c156e917f52ffa704cc151fa46e1e328673d","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j7570432-mgmt-2\",\"clusterName\":\"ci01-j757043","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-25T10:49:06.755889618Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-25T10:49:13.818157295Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-25T10:49:14.199363511Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T10:20:52Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-25T10:49:06.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-25T11:19:06.506Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-25T10:49:06.755889618Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-25T10:49:13.818157295Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-25T10:49:14.199363511Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7570432-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7570432-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7164/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103429520567570432","occurred_at":"2026-09-25T10:20:52Z","signature_id":"4c0157b6486aa8ba13127ff990f84148fa30b014d23cc1a62b37dca7e1e44aab","pr_number":7164,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"70097b1d8f359ab8e0f51f7aa2fbee8af0018247211e97b38af99aca36140186","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j8724736-mgmt-2\",\"clusterName\":\"ci01-j872473","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T11:23:53.695872086Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.867241647Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.995607642Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T10:58:09Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-22T11:23:53.670Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T11:53:53.299Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T11:23:53.695872086Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.867241647Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T11:23:55.995607642Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8724736-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j8724736-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102351739108724736","occurred_at":"2026-09-22T10:58:09Z","signature_id":"2feb22461dc45624d789f70eb42c73cb09a2352e5d415e8876b9a64e3920abfd","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1f0d1c0d9bb7c84ece352b5ff00277205bc7b31c723e9e6b1adc5bdf14ad3d00","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j9153920-mgmt-2\",\"clusterName\":\"ci01-j915392","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T07:33:19.023908785Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.03768985Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.167202045Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T07:01:28Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-22T07:33:19.014Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-22T08:03:18.587Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-22T07:33:19.023908785Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.03768985Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-22T07:33:21.167202045Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j9153920-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j9153920-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102292246979153920","occurred_at":"2026-09-22T07:01:28Z","signature_id":"de77c358890614e2ec342728c508f5e003c54336cd8a63368d4ae3d9e2e1bb39","pr_number":6869,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","failure_pattern":"expected all guest nodes to be unschedulable, schedulable=\u003cnode\u003e, \u003cnode\u003e; nodes: \u003cnode\u003e ready=true unschedulable=true created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=true unschedulable=false created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=false","search_query":"expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T06:49:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","signature_id":"a02ab509af0dce2add587b938aa804dece2558ca7621ffc208e009f99dae4089","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"58103f0bb0f23a6bcc7180fb82d4fd105caee89b2c2667152f6ec564d76dd64c","failure_pattern":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-\u003cid\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish: context deadline exceeded","search_query":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T14:55:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"69e234e12ff1dfef8a8e49d1251b6e3da1b82338b95055a0f148ecb1c35fafd2","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-24T00:16:20Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n  \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n  \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n  \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n  \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","pr_number":7119,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2458db19d1aa1c830d2005883aecaebf402e5888aa2957a0d46c624d33b2bd43","failure_pattern":"failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s","search_query":"failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-b6k7qp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000722f40\u003e: \n    failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-b6k7qp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000722f40\u003e: \n    failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","signature_id":"632ad063d78546b35d3640152b34002f194bf542539b83c541017997e134011f","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca45167e71db14f5aeb17f2f3385a1b55e20eb87859fa05cf7e6c97421bb62be","failure_pattern":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token","search_query":"failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T15:00:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 623.703s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c664c0\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token.\n    Attempted credentials:\n    \tClientSecretCredential: unable to resolve an endpoint: server response error:\n    \t\t context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 623.703s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c664c0\u003e: \n    failed to gather logs: failed to execute cluster id query: failed to execute query: failed to execute query: Op(OpQuery): Kind(KInternal): Error while getting token : DefaultAzureCredential: failed to acquire a token.\n    Attempted credentials:\n    \tClientSecretCredential: unable to resolve an endpoint: server response error:\n    \t\t context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102412736506892288","occurred_at":"2026-09-22T15:00:31Z","signature_id":"ddf39fe4612073dbe574c94e7b39f5362ea6232c9183ce5f9846126b8b3b5e3b","pr_number":6756,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2553fa7889a9af1fa3b8eb97a5f22ebd02ed4cb66f230b2442c0efe51892094d","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": net/http: TLS handshake timeout","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T22:18:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d70460\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d70460\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","signature_id":"d790667a42bc853d57fab4f94768ab9d93901fef5701562ee2de998ed5c7396d","pr_number":7148,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5a32867416c248246850f053eb447f1cb93d8fba2bae3455ee41d0153f99c5c0","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection reset by peer","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000858480\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": read tcp 172.24.8.136:46186-\u003e54.82.178.161:443: read: connection reset by peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104335322849480704","occurred_at":"2026-09-27T22:20:12Z","signature_id":"e8a205541114f6cec3b425e2ad1adb4dc2be321629c4a93557e1f3b4a5460bb4","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38","failure_pattern":"failed to roll out Helm release: \"fleet-reg-ci01-j6376960-mgmt-1\" has no deployed releases","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T17:49:53Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-23T18:34:52.490Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 values=\"map[cloudEnvironmentName:AzurePublicCloud cosmos:map[name:arohcpci01-rp-j6376960 url:https://arohcpci01-rp-j6376960.documents.azure.com:443/] image:map[digest:sha256:6bc5ed31e42afccc2abd6b21b817cc402be829c20f8cc00e71fde01c9be1523a registry:arohcpsvcdev.azurecr.io repository:ci-op-6hlkxni0/pipeline] registration:map[aksResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j6376960-mgmt-1 autoApprove:true hostedClustersManagedIdentitiesKeyVaultURL:https://ah-ci01-mi-j6376960-1.vault.azure.net/ hostedClustersSecretsKeyVaultManagedIdentityClientID:b2cc94cd-3e8b-42a9-bf2a-e025837a95cd hostedClustersSecretsKeyVaultURL:https://ah-ci01-cx-j6376960-1.vault.azure.net/ kubeApplierCosmosContainerName:Manifests-MC-1 maestroConsumerName:hcp-underlay-aro-hcp-dev-shard2-slot-01-mgmt-1 maestroGRPCTarget:maestro-grpc.maestro.svc.cluster.local:8090 maestroRESTAPIURL:http://maestro.maestro.svc.cluster.local:8000 provisionShardStatus:active publicDNSZoneResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960/providers/Microsoft.Network/dnsZones/aro-hcp-dev-shard2-slot-01.hcp.osadev.cloud stampIdentifier:1] serviceAccountName:fleet]\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:34:54.319Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 namespace=fleet\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 dryRun=false\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:35:05.401Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 numVersions=0\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"\\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","occurred_at":"2026-09-23T17:49:53Z","signature_id":"aabbcc585420cff60362100bdaf09336ff0307d5c296c6855ec0f5fe21f850ce","pr_number":6915,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-17T08:37:58Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-28T14:59:06.467Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-28T14:59:07.052Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:bd6cc099-ebdb-45c9-b00c-0802e69adff6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j9638656] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j9638656.hcpsvc.osadev.cloud] image:map[digest:sha256:e46e58946095455f1186c85136cc576df37e30746f4cbf6630f40ee15e5c51e6 registry:arohcpsvcdev.azurecr.io repository:ci-op-w348pxjm/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-w348pxjm/pipeline:pull workloadIdentityClientId:78d6364f-6f7b-4fee-9060-b833e4581820 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:85e8e804-1ab2-4c98-b7ae-b44472a84e8a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-28T14:59:07.052Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:07.261Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-28T14:59:07.573Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-28T14:59:07.573Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:07.688Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:15.529Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-28T14:59:15.529Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576804189638656","occurred_at":"2026-09-28T14:19:37Z","signature_id":"aea56e1c6bff3ce711725997c0e6216095b05913193456533d76f637a946367a","pr_number":7203,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478","failure_pattern":"failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T13:01:50Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1}],"full_error_samples":["time=2026-09-22T13:40:24.327Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-09-22T13:40:25.067Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpci01 kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:9e63919b9dde3a44483a565c55744abf9eab54314586f01a631a42b1104d5f2a] metricLabelsAllowlist:[nodes=[node.kubernetes.io/instance-type,kubernetes.azure.com/agentpool]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[infra]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:be4e1b8bf4b8cc868cfa993818ed94e2536adc15713fc64942ce6a47d1bbabb9] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:b840d65d98ba99f4d62329db721714b36ec23d7a0c8721d1fdb4fc839f97cf76]] thanosImage:map[registry:arohcpsvcdev.azurecr.io repository:thanos/thanos sha:b567818fe608067eb0f1d7c2c4fe361e7ad83c8a256234c97685f1d0bf670cc8 tag:v0.41.0] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[serviceAccount:map[managedIdentity:d7ccda51-6374-4c36-847f-8c68f35d13dc]] prometheusSpec:map[externalLabels:map[cluster:ci01-j6088704-svc environment:ci01 region:westus3] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:92fa8f5e91a0e02e0107c346f6c66657738dbf394946e52c208150fd498554bd] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-ci01-j6088704-svc-ybo2.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-7b09a45e774144ada99a9dc87a23c02f/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 replicas:2 resources:map[limits:map[memory:NONE] requests:map[cpu:NONE memory:NONE]] shards:1 version: zoneCount:3]]\"\ntime=2026-09-22T13:40:25.067Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:40:25.331Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-09-22T13:40:25.636Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-09-22T13:40:25.636Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:40:25.815Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-09-22T13:46:35.923Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-22T13:46:35.923Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7066/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102382882726088704","occurred_at":"2026-09-22T13:01:50Z","signature_id":"6bac7c9167143bb4a1dfabfc4a6d81083bef0f715dcabad843cd680ef59999d4","pr_number":7066,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","failure_pattern":"failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T15:35:15Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T16:33:44.590Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j1695232-mgmt-2 enabled:true environment:ci01 image:map[digest:sha256:21861cc03f51cd5c5fcc5f37f79aa945bfe633b1fbdb5c662d6376edb84a8a45 registry:arohcpsvcdev.azurecr.io repository:ci-op-s7p7jyyz/pipeline] region:canadacentral resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.293Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.516Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"feaab49de56d93336deae269ed4dd6eb946386d142e5a85a65c80e15acb2b43d","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"54c2b034583e4a5d24eb9d13c73dcfe6a50efc774d9a44838d433b2a32ba910a","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T02:58:44Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","occurrences":1}],"full_error_samples":["time=2026-09-25T03:43:40.223Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-09-25T03:43:40.544Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m certificateNames:[frontend-cert-ci01-j0816768 admin-api-cert-ci01-j0816768 sessiongate-cert-ci01-j0816768] clusterNameFilter:ci01-j0816768 clusterTypes:svc-cluster,mgmt-cluster collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current,keyvault-certificates image:map[digest:sha256:742269f198294ce81d1f0d1f1fa3db44028559acb86d454e16d8997543a54aa4 registry:arohcpsvcdev.azurecr.io repository:ci-op-01nc500m/pipeline] keyvaultDNSSuffix:vault.azure.net keyvaultName:aro-hcp-dev-svc-kv kustoCluster:hcp-dev-us-2 kustoRegion:East US 2 listenAddress::8080 region:canadacentral resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:128Mi]] service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:9cf3e005-99aa-4353-992d-fc05ca747f7f name:aro-hcp-exporter]]\"\ntime=2026-09-25T03:43:40.544Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:43:40.690Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-09-25T03:43:41.003Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-09-25T03:43:41.003Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:43:41.052Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-09-25T03:48:43.009Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-25T03:48:43.009Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp-exporter/aro-hcp-exporter not ready. status: InProgress, message: Replicas: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103318256310816768","occurred_at":"2026-09-25T02:58:44Z","signature_id":"600bfb947207083afdd67c9ce6feb6937313b5ee52299c05181ebcb8242f4515","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-22T06:52:43Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-09-22T07:21:38.848Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-22T07:21:40.354Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-22T07:21:41.548Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=d03eff9eca8ce6668b3735feb0fed36048a18f0177222fc337f65cc4e325d887 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4723968-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd03eff9eca8ce6668b3735feb0fed36048a18f0177222fc337f65cc4e325d887\ntime=2026-09-22T07:51:38.563Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7060/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102289975184723968","occurred_at":"2026-09-22T06:52:43Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":7060,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","failure_pattern":"pod targeting cordoned node \u003cnode\u003e should not be assigned a node","search_query":"pod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T06:21:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n    \u003cstring\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n    \u003cstring\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103006860184391680","occurred_at":"2026-09-24T06:21:22Z","signature_id":"613a9b7adaccf1b621c4d00e5e9f881ac489c8f71ae2c82dddfa7345990fdc47","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-26T16:16:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103881405791473664","occurred_at":"2026-09-26T16:16:29Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T00:11:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc001758f60\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc001758f60\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"9fce1ec92cfb95fb8f8175f8912efb491fd5065485198557528e2813ccb986fe","pr_number":7112,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","failure_pattern":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T20:01:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","signature_id":"05cb3f86a6f99432ca7873642f1755e0d1de74704c579e10519fe31b0a3ab239","pr_number":7110,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f76baa3d4767ea148b5500ace79ced7c3f0e0092f22b03bde161fd5f0c1796ae","failure_pattern":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-26T20:17:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00107dc50\u003e: \n    run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 608.861s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00107dc50\u003e: \n    run command \"e2e-runcommand-1790458991773693237\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103942172993654784","occurred_at":"2026-09-26T20:17:58Z","signature_id":"4eac6d1d6c768c796dcee39de9244b0faf13f00b731ed7de804c181901f6e831","pr_number":7068,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","failure_pattern":"scheduling requested CPU must be positive","search_query":"stamp 2 scheduling requested CPU must be positive","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T12:46:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 2 scheduling requested CPU must be positive\nExpected\n    \u003cbool\u003e: ...\nto be false\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 2 scheduling requested CPU must be positive\nExpected\n    \u003cbool\u003e: ...\nto be false"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102379019017129984","occurred_at":"2026-09-22T12:46:33Z","signature_id":"7867693174b52991f98e5b358683d9b1a30f9bef5582588f00189e01aa4a23b6","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620","failure_pattern":"{\"level\":\"ERROR\",\"msg\":\"run failed\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5982976-mgmt-2\",\"clusterName\":\"ci01-j5982976-mgmt-2\",\"region\":\"westus3\",\"err\":\"waiting for deployment","search_query":"{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.14144271570014144,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-23T19:42:53Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-23T20:05:19.475Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-23T20:10:16.316Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-23T20:05:19.482013132Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.259341341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.679397764Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314281955Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":\\\"Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/deployments/aks-cluster-create/operations?api-version=2025-04-01\\\\\\\": stream error: stream ID 41; CANCEL; received from peer\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","occurred_at":"2026-09-23T19:42:53Z","signature_id":"b83201cdb7e435a0abab8639b944505f58098f612295a321d14698e824fe6511","pr_number":6925,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":30,"failed_runs":11,"raw_occurrences":15,"matched_occurrences":15,"runs_affected":11},"rows":[{"environment":"int","failure_pattern_id":"40b6645c8d06a920c8583c663019b4b6b7fdd9c77dba9f0d00c2d04055f84254","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":10,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,4,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T15:01:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000599de0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n      \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000599de0\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-kcj4l5\" in resourcegroup=\"rg-np-skip-minor-kcj4l5-45s8nhm4k78b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"name\": \"da564fd3-aea3-4a41-b171-3969372d023d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:39:04.653915252Z\",\n      \"endTime\": \"2026-09-25T16:58:09.716908828Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kcj4l5-45s8nhm4k78b-np-sm-kcj4l5--managed/providers/Microsoft.Compute/virtualMachines/k7t8o1t6u7r7f0q-nps-420-422-lf9bj-qzzxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n      \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00075fc80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dnhw57q2zhsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"name\": \"9cb82bcd-1a84-400b-ad2e-8cf3dfa50902\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:30:53.138668141Z\",\n      \"endTime\": \"2026-09-25T08:49:53.966422643Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-dnhw57q2zhsr--managed/providers/Microsoft.Compute/virtualMachines/q8f0o9t2z5z1x2r-np-1-9lssw-jg4gh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n      \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c02c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mw49n77wnb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"name\": \"e57de6e4-350f-44ca-8989-0fa815b3fd2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T08:27:10.94071903Z\",\n      \"endTime\": \"2026-09-25T08:46:13.959695236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-2mw49n77wnb2--managed/providers/Microsoft.Compute/virtualMachines/y5d4i4z6u4m1b4g-nodepool-128gib-j5q6z-kjlzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","signature_id":"2374477c920f24f39cbd859778b4b5423d9aa24bc3495ca463c798a21c8c8e87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","signature_id":"4b38b160a22968120417e984b979b00dd8bb78826d4b38714c0eb589ea58bd2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103397522356572160","occurred_at":"2026-09-25T08:13:43Z","signature_id":"b009af289e732a1b6a4d96b5624455888514f82524439fc4e668740147e9e47a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103303905172525056","occurred_at":"2026-09-25T02:01:43Z","signature_id":"f35b530f26f0d3fedd585cc4516c1a10ecb44588c174c5fe5a05614b69732b79","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-27T07:04:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00024f2c0\u003e: \n    failed to create NodePool np-fnv7lb: failed waiting for nodepool=\"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\" in resourcegroup=\"rg-candidate-5-0-v6nvmt4rxzjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\",\n      \"name\": \"203ad465-1e1f-4337-9659-f5fe2972a72a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:17:06.138034027Z\",\n      \"endTime\": \"2026-09-28T21:36:06.809868093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00024f2c0\u003e: \n    failed to create NodePool np-fnv7lb: failed waiting for nodepool=\"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\" in resourcegroup=\"rg-candidate-5-0-v6nvmt4rxzjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\",\n      \"name\": \"203ad465-1e1f-4337-9659-f5fe2972a72a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:17:06.138034027Z\",\n      \"endTime\": \"2026-09-28T21:36:06.809868093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116fc20\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kj25nlz5xd2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"name\": \"f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T07:29:57.086711017Z\",\n      \"endTime\": \"2026-09-27T07:48:57.003548378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00116fc20\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kj25nlz5xd2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"name\": \"f3fb7911-3b45-4355-a4ce-119b44f62f9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T07:29:57.086711017Z\",\n      \"endTime\": \"2026-09-27T07:48:57.003548378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-kj25nlz5xd2c--managed/providers/Microsoft.Compute/virtualMachines/j7g2h4t8i8p2b9x-arm64-vm-np-1-h8dgh-s6v4c to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104667201544916992","occurred_at":"2026-09-28T20:18:58Z","signature_id":"1e10870f9142e34bc1040827a466038cf22783d022da97c635df733bee6dca9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104104984143466496","occurred_at":"2026-09-27T07:04:55Z","signature_id":"931488020c0f76286d54766d64d2f2259ec91252cd34366384a920f857d3befd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-23T10:44:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102710804108283904","occurred_at":"2026-09-23T10:44:56Z","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102650382160236544","occurred_at":"2026-09-23T06:44:51Z","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"38eeaeccffa0491180dfc6054cc1b3abf572643c67dff782fd1fdc281074fb46","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T19:50:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000914940\u003e: \n    failed to create HCP cluster ea-list7cctv9: failed waiting for cluster=\"ea-list7cctv9\" in resourcegroup=\"ea-list-wc6fbxsd9rxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n      \"name\": \"c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T21:22:56.593640254Z\",\n      \"endTime\": \"2026-09-22T21:51:58.372098632Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m51s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000914940\u003e: \n    failed to create HCP cluster ea-list7cctv9: failed waiting for cluster=\"ea-list7cctv9\" in resourcegroup=\"ea-list-wc6fbxsd9rxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n      \"name\": \"c3a3e500-1ba1-4cb7-8434-bc1aa96ee070\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T21:22:56.593640254Z\",\n      \"endTime\": \"2026-09-22T21:51:58.372098632Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m51s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-h7c9rnh9n24x/cluster-candidate-4-21-jthhdc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a600\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jthhdc: failed waiting for cluster=\"cluster-candidate-4-21-jthhdc\" in resourcegroup=\"rg-candidate-4-21-h7c9rnh9n24x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n      \"name\": \"b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T21:07:20.636049246Z\",\n      \"endTime\": \"2026-09-22T21:36:27.958431734Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m58s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-h7c9rnh9n24x/cluster-candidate-4-21-jthhdc should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a600\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jthhdc: failed waiting for cluster=\"cluster-candidate-4-21-jthhdc\" in resourcegroup=\"rg-candidate-4-21-h7c9rnh9n24x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n      \"name\": \"b55cb22c-9a55-40c5-b3c6-893ec03a9fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-22T21:07:20.636049246Z\",\n      \"endTime\": \"2026-09-22T21:36:27.958431734Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 2m58s ago; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","signature_id":"4fe90f7c3f08ac9dfbdbb935dc163323197da02f18ba46153bdcfc7eba9b8d77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","signature_id":"6299df96b3ee9add016c9b5bb17a4bd6b607c208223caa2e182fd6c3ce6bd576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"46b0598b5ebe8f06dc5f7cfd83233a53219d8dfdc2ad61e72855646ca0260a66","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T05:49:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c10900\u003e: \n    failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c10900\u003e: \n    failed to create NodePool np-pt5qmx: failed starting nodepool creation \"np-pt5qmx\" for cluster \"cluster-candidate-5-0-pt5qmx\" in resourcegroup=\"rg-candidate-5-0-9clwz8rf6599\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-9clwz8rf6599/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-pt5qmx/nodePools/np-pt5qmx\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103361291237199872","occurred_at":"2026-09-25T05:49:44Z","signature_id":"26173be16e73b2272c2dda84a4b010bd8ef49864c818286cf90989c383c22b93","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-26T21:16:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d427a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2f88crxn77t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\",\n      \"name\": \"b648d347-dd3d-4511-8fd6-84bc235918eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:43:43.636760265Z\",\n      \"endTime\": \"2026-09-26T23:02:47.067520117Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d427a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2f88crxn77t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b648d347-dd3d-4511-8fd6-84bc235918eb\",\n      \"name\": \"b648d347-dd3d-4511-8fd6-84bc235918eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T22:43:43.636760265Z\",\n      \"endTime\": \"2026-09-26T23:02:47.067520117Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4p0g2x0p8h2h6n-autoscale-noaz-ncv9k-skqk8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103956778688450560","occurred_at":"2026-09-26T21:16:00Z","signature_id":"10d0c74d7821e472e9f3cd0ab2d72680f3767a27a31d204abf40c872c382c5df","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"53d6a9f6efeca45a9da0e1f8eb60e93e70de646c11d05978730da3382fb8e964","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T15:01:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n      \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e80\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-p9kv2wz66ng2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"name\": \"82d3308e-222c-40d8-a1be-8c057c26bab1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T15:45:43.486812846Z\",\n      \"endTime\": \"2026-09-25T16:14:49.560161271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103500347635929088","occurred_at":"2026-09-25T15:01:22Z","signature_id":"e1f4ee9af599a77b687540e9f4697061e341edc13f3416df00ca7e8b696944ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"9bf4f3ad9b586932b28d6b547ac6a878bc489620fc6853c6e80a7ffdde8e88d1","failure_pattern":"VerifyBasicAccess failed: failed to list services: Get \"\u003curl\u003e","search_query":"VerifyBasicAccess failed: failed to list services: Get \"https://api","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T14:57:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00158c4c8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00158c4c8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103136634483511296","occurred_at":"2026-09-24T14:57:02Z","signature_id":"c6250a5a334f195a565bf422264939eadf384e54a5927ab71926e7b2cd23c48c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"351f45356c048233596e7dd87add23f59e5ae9c144a3196c0577133fac0ad667","failure_pattern":"failed to locate cilium chart: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e: connect: network is unreachable","search_query":"failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.3333333333333335,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T19:50:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:159]: failed to install Cilium chart via Helm\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008581e0\u003e: \n    failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:159]: failed to install Cilium chart via Helm\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008581e0\u003e: \n    failed to locate cilium chart: Get \"https://helm.cilium.io/cilium-1.19.2.tgz\": dial tcp [2606:50c0:8000::153]:443: connect: network is unreachable\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102492733552726016","occurred_at":"2026-09-22T19:50:05Z","signature_id":"6436a970a48f30cfe1ee0c5e3a5720711b6021d2b00089bf5fd8c321a3203f96","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":28,"failed_runs":21,"raw_occurrences":176,"matched_occurrences":176,"runs_affected":21},"rows":[{"environment":"prod","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":37,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.857142857142858,"anchor_occurrences":37,"after_last_push_count":37,"also_in":["STG"],"trend_counts":[0,1,1,1,0,6,28,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026392a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pndxf7r8t9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"name\": \"a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:40.174684426Z\",\n      \"endTime\": \"2026-09-28T00:42:43.628005146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0026392a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pndxf7r8t9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"name\": \"a91e490f-050a-4c07-a58f-af347cb9f143\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T00:13:40.174684426Z\",\n      \"endTime\": \"2026-09-28T00:42:43.628005146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866340\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wg62gnrkcq82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"name\": \"38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:28.308542215Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969190413Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866340\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wg62gnrkcq82\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"name\": \"38c68d19-2950-482f-9bf9-f70f6c7063a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:28.308542215Z\",\n      \"endTime\": \"2026-09-27T23:31:35.969190413Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992660\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nwqdmg89c6p9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"name\": \"7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.690582806Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971437914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992660\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nwqdmg89c6p9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"name\": \"7c1f6eed-e1f4-4411-9a1f-fe2e5ddf1e50\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:36.690582806Z\",\n      \"endTime\": \"2026-09-27T23:31:45.971437914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"03d18401392997c3274ba2bf78e810311803b87bc7892a064ed600daf1058f86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"0ebe8a5d8973b947fbdd076fb748fedd8a7ffc4012adba74283db57c6063bad9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"1ab642fad09693398e82235c858d45a1797714580df61abc6d1b19d3918b71a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"1ca48fafa9f0edecfb3787a894dc476ef91d87d36f79d5aa9e05a3fa12000657","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"22046caa0147e84e4bf00172320247011b3cb7426164354cc5c2ff6648d4106e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2b8afe759c56d8a7a8c15ff5b208b75d9c57c67b66b8b03a83b7df762b3540ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2bee03d5fc4053026371b7f338b599922d58ca513c59a44839dfe20827418271","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2ddf0eb41f7f4835c32a0934a2b734e3cff4bfce66b8ba7fa3315db46958dfc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2e82467c69bc3d4c0bfce22bf493cfb693f196befe788f76a6d3957522e3f5c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2fc187761095416d0b74514b2e03a845bc6e9789fe8167aea732a57b59785d7b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"33bb4455c170f5b024d933dc8204c4aa021b105023501e716ca4e138345d98ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"461696ab84dd3f6f5d834c5c1b6fcf988c3286f80243cf0e4e0b3877884cad1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"5853de0191a9e066d96eb6447d7e06566e036f05860c98d4f1c042f9f060842e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"5f5245c97fdbd17ad8ac15e3e2445694b4f8d2a8f15128ca7457652d20c85a4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"60bff82014486820406a93416912dd354a1c71981703f4a298836390037dba3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"63a069ecf46f44ce769fd7002561795b40f053bb56e3e21dd6e759403fa2f18c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"64d463d42f7127948f5a0d3f204437431ff48e5601f36b43f3827e98c0d20698","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"650a5236bfcf5b14da373d69e1a2c8e2d365bee0c47a5d780593d397e4c57145","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"65288eca0965be6dbcfe206e0aca14c0c0ca1eac6d17fd4f41aea772375b869d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"6ae5ca364d6100ceef84da7c983fcde591de89d92c71a98ee689f822e2415b43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"6f750e9463768b9049eeacb07c97390e2b8b9fff31ce5357d95b8e0b486676c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"ad9b6981d10e4fb3c5ba70acecc09e110752fb556f8b94cbbd533a7e4f924b6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"b94e5cf2e7c1fb55ced86979ddc7046ca3a21ac3308be4a35685174df4fcff94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"b982657b8b51d839bd1a6e80552bb477e8899a650c3d88c5037ab958c4b34126","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"c41ccfc8ae07559bda7c04e5a227db6555888492c62d50c7b225a44f527e69aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"d60bab05ea469af575a8a75ee3c255b1b84c6cf63eeb43e333195545bae9492e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"ddc46e7b679376d943aca50730e23da423333de42aa64aeea50b2982c7c8b320","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"f9162987953ca17cf343f465cb28a946cb34440c9d858e66ef18592f3f2b461b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"15b134d9b1c2f11953fb29588acde1168f9064424aeaeb6430679ae6824b0421","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"2a779a411bf593492105316539abd1f54e91aa141745a5334ebf3964c0c8ce1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7882bf4b3dcb83a7eeee2c65b5637a07538c258541b84315afe9977667717afb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"a998cae1ca6926e3ecab46ec3a7d97e743d778004a1cecd37cc4592f661d0b14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"c87c45b4bc08cbfaaf0b572926aa9d17b34835b20fdb957c54f446329a117b21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"cfcce88484594d3941b2e7b054c3d0cb474628fbd9bea67f20782799fb189503","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6d276974ce3c3d3fd4a182577c36d577de093c2c2e4513abb2edf58915543439","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"af6920bd4dcef413e75f2c2ee8ed9b6ea9da89a5c3cbc85c4911bd464e6f591d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"32e3a94a639de7d26ad7cb7a82730cf1fe540c3bacab33ab7a4b65d652194ad6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":29,"runs_affected":6,"failed_runs":6,"run_impact_percent":21.428571428571427,"anchor_occurrences":29,"after_last_push_count":58,"also_in":["STG"],"trend_counts":[5,5,4,5,5,5,0,5],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"6cce0979bce816f94e1a76410fe9b0dfddfaeb9167c8d068f6c6c58b8957a7db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"7732b8465eb0b310323c46363238564ae3708ad1dd66f83e67d67744e2112be2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"9aafea9b5d579d12248bc117bbaa084f2543cc7e5eb735c49007499bd1c1c528","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"dd38e1bb9f1353d3c3ede4dc31e69fe80aa02069128581607c7d92391c522e32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"f675dd7539b7a7c84d8073b82c8ff50aab68a3afafa2637412a75cd680bb8823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"07ee127f2734c717db5d7963ebab6a05f82747a01086f95b5e8522d1bea8a1d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"312d047aeaf9f27a3dfa801524792c6c23a767503bfb0fdd7e4db5a9391834c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"3c227abf2120bdc1c31c60ce1f1703129beb844d188f6c346f3c5ef45d06f46f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"a29c2cf5154fd0102d2696a881ff253f543b613779166c61baa2d90666be4f55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"b4c1f426afc58efe4dfc7270dca2f6ec6e980249e58e66d3b4ad7311b8dcf458","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"06fc4248c731a78ec09d70b92471beff4048830ea286a825e3ad2027e0ced359","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"0da3e1c733b3a346e6237f6eb60d4c8b4b0a1f2ab3382bc7c6e0a96d1c2b7c4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"1323fc4bbb4f8bec0749ff5a18b557ada5537b1e2d9ac0d4f832c4644c0b23b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"2d0664bc68395570897d3edf86ebea891697dd9f67b9c208c184ac4acd358176","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"76257e848298b1062f3dec33bd08c56cb39fdc88fccdde36fe3867b8cef0eeff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"0856a5b7612ceaf27aef57335208fdb96fec3578c31cded0af090414cbc459ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"163ba41eff7851369f9490bc51639528b5ab45ae39f71f38a3bbb8b825ec7e1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"369b11784caf993a1266d4a2191af0129727f06deaa23b4c15d665538f183d95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"39cc51f3ed9337761bd1c79d82ca52214f6889badaaab5d696c36fd31637d8b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"78618845cda7ac86dc1a7ff167f19e01de1f527c7ec62a7cbacf618e8f8a10e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"0b19163b0950f2bc9dac2fa7bbe2fe9b7e8b4eddb5161aef3f7f5ec4427662e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"3532da433aa865198e7af69b2d67f9f685db4cc1befc31928db022dff5c8ba6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"4beec4b23a49b2c4afc633ee4de74ca68b43b8c0a169e74f6e8c71ff427a668d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c0473fca37809ede533480e60205fed102ad16eccbd4d3d6441046e31b66bf9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"2d6b9a10fa16a43e4443cbb8ae50d90afa7db260b400159e32bd31c9c06e8fa6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"589aee349dfeeb917b1e915467908bfe001aded4ad5fc993461f2c4f097923d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"69994bd3122cfdf5cead21810ad001a53d6fca7838b640c3b6a6fcf65c3e702e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"8165a74a386574cd51eb628b8ac81aa0881833bc8abe946100d3ecc127fe774a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"93b07e54f382d1b86bb934122cf1eae24d4dca6e7c4ffc2ab45db781b7040196","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":26,"runs_affected":7,"failed_runs":7,"run_impact_percent":25,"anchor_occurrences":26,"after_last_push_count":46,"also_in":["STG"],"trend_counts":[4,4,4,4,4,2,4,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":7},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"56f51dbe4172c7698c7e2ac9b44cc9309db978fcf5349649586b91b53538f803","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"5e02d079f2bb2b4baf359fdcea15f52be4ed619abb6034bc78cd714b27a3f342","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8bfd58413915673e86f8668f73c4a1593d6b25ac14d3df8cca7f16fa47901f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"c36a633bcd07dffb48200bace887b57ae6fe20eba4fb59a0da31d14814ce0b59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"7026298ac0b13e5919540ccf258871ea869dd1bad766be2b0773591c9914e4b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"a9e1b9127914b42c7adad7624b11d42e888367c10a014dfe93150ff03893a4cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"cbfeba8437d8617dc3f95e1a802b22d1f2cc72e90e32e533cc4a5aefd6974feb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"fb749055d31dcc0dc83a0c66aae251a4681c7b41862f59b30547d03392614d99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"e47e5f96cdb4ed158c7fbfae7715d756b161b0ff44ab47008e6af6e7d32e634e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f793a0f121645a3b3ada14d1116f5029bd947c64fbdb8bc72281abdef651667c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"25c9c8629d36608562eacc178345af00071b7c15f03f2b7932324336ce809df8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"5b65b5d543e8d31941a31a2864c015c8c52f0f301461e1c272635b2b14df1af9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"d0fabc45709d827c15fe3c2a6fbbced1e9f6e7903fe8a15397d348a35770aa86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"fe9849f3545963512f1ece2f25d17c32ca1e7e435e9e95ea90adfb3ea562ce8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2c2448420f0585ced642c7a920d0fe02a87408a0b9d29f4ef0081d2ced376382","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"62770c4f0e621d8962a43577db753baffbfa217cfa5c51a5444cc20ba8dcbaa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6e1813acc449a6535ee059b4dc15d561d04689edaf2ac0927414e732fa390dd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"7e29c61c5ec9623a1d7631cc8d8c2bf09662bec493f2b1e0c6761a8e6df9e943","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2c73a06cdcacfc08c5ae0807302796c3e3154da70a86dd6ed16c84098e26ab85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"57f75e9006bbdcb6503c784e4aae06190b496bc4d2463173bd66d696b13fcfb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9fb7620a84e0a14940bbc581680a78cbf6f7134a9005afd5f2451b87d1987486","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ccd3082ac0ec3ee7f5872b88cabf71e9eaf1a7b9a9598367688dc3bb14866a3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"2553e871dbbe3bcfe30dc281f555126453c37b319bd95a2ea76f0bd2cd8c9c80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"45414bb4c5c8a5a0bc7c8f0b4b63112f040f98363e403c86a8c64729cad56efa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a152d3d30eea393d48798792d545aabeaefde64210923bcf1f0905915e39788e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"d60ff50f583519824d6d7e580f49b3097c053eb851455b216fd32931071a2b74","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":21,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.714285714285714,"anchor_occurrences":21,"after_last_push_count":21,"trend_counts":[0,0,14,7,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-24T02:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0f42ca5bdd5424ebd462017e995420f979246e19a7f68c1a45f2e182770b37a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"3c9ac5911363cbb207f63f2d230928ca57009144d5f790aadca90edd77cd8e2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a043abfb4a137ba5c7fb99a1284be7cfe198204b214476f6fd57e00fee845d3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"dfda133e06b8ddc58347c148dd004db522d35f830c18a3e53b5d3cde82d08ea0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"e328078175fb41c8dfbd4578e31c90a7c72998a5382576bff48a3e7cd16f791f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f1db6949bdfa0c436485b175eee7c33e360c184472692055a39f14766a6685ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"fa688a48213278aa49f8f4617b7ea8d841d18133d8772e5ee0c28401f6d5f777","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"696c58cbd3aa8d300062df4a47583e3f809311c672a43e3da76b7164514ff28c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"80af8612530648ca50b987b6f9cdbcfec5e5733a4eb73b613f4f2cca40440dda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"8b972b568eeb592cb7c4e3ea2dd32eb95af26dd5cf16683b957140e9cded71c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9099db8102021ee82d4f7fa885ff9d6c40c7a777e05056ac01595d8cbe6b71e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a15265dd30984b38cf2e97f6a7e9fa79ca18ae3cedd2d533ed100ef489695a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a6b4d2a29f145149ab937c5195b130875034ebc08e0235a73002c81ef5d5096f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e5a3278c01c0abce683e00630bbc784dbd610e4e57d5f23368cb17f09c05ad78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"3c47413f0338789d70091286ea56544fcdee01d67131f0f7851a0b97474d7105","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"60163a0f600eba5c06ff2c878eb2a0092d005dbebf2c0e82fe91feb85ffe2ec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"950be739fb33f7b6adecbfc5254b9a9b8d54463fd0cd14ad47b8f998fd5afdc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a1b9cf6305c1912800fbad4a1b18908456f24a8e3cd6243102d4426c2fb0976f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b2a0aa0f0d5efc781c39edb1e340969ee91b48c4052331e8dd71b8357572e91b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ef884a233cc88172af6105a108535d3de27389d5003c528f736c27bc0329a063","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f40ba7ba94d5d5cc9b8228f74ef22fcc6c8cf1de075f86ae5b726e2a4ba76bcb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":12,"runs_affected":10,"failed_runs":10,"run_impact_percent":35.714285714285715,"anchor_occurrences":12,"after_last_push_count":28,"also_in":["DEV","STG"],"trend_counts":[1,1,2,3,2,3,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":20,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13b60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-6k9kth8pvmf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"name\": \"eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:13:34.441508798Z\",\n      \"endTime\": \"2026-09-27T02:32:39.228262555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13b60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-6k9kth8pvmf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"name\": \"eb2dcee5-7fa0-489a-bdf1-3568cea3218a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:13:34.441508798Z\",\n      \"endTime\": \"2026-09-27T02:32:39.228262555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138eac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vn4grw565jrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\",\n      \"name\": \"7f58a11a-50d1-416b-9f82-356e8273b038\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T23:14:10.630574624Z\",\n      \"endTime\": \"2026-09-26T23:33:16.736724619Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00138eac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vn4grw565jrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f58a11a-50d1-416b-9f82-356e8273b038\",\n      \"name\": \"7f58a11a-50d1-416b-9f82-356e8273b038\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T23:14:10.630574624Z\",\n      \"endTime\": \"2026-09-26T23:33:16.736724619Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010653a0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5jg8xwpwj9d7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\",\n      \"name\": \"a82865e3-4228-4f7d-bc65-f5af89262225\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T23:13:30.607288907Z\",\n      \"endTime\": \"2026-09-26T23:32:36.744317191Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010653a0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-5jg8xwpwj9d7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a82865e3-4228-4f7d-bc65-f5af89262225\",\n      \"name\": \"a82865e3-4228-4f7d-bc65-f5af89262225\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T23:13:30.607288907Z\",\n      \"endTime\": \"2026-09-26T23:32:36.744317191Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104028667012190208","occurred_at":"2026-09-27T02:00:24Z","signature_id":"ab3a35949bad5bc410ebcfaade95c153a7cb24c117efaed240cca8cf81980e41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0fe18cdd8de62f844def4fa357318635d54320867a1c8633ec2fece9565ce3ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"1f3b4bd3b5a7a721eff71d604efa7685fa048deab8b8cf0fe82a80b643595100","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103665929874313216","occurred_at":"2026-09-26T02:00:16Z","signature_id":"1ed3f84c5538fe158e1ea52e2cae1b12fdc1a35c9a28671ff845649d33b5b757","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","signature_id":"e07f78fb07fe2b29f2a237f72f104117e81b0766f04a34ec1cc5af886ce79354","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","signature_id":"f5bee8b440fafcbcf3226fb257766f44fc56668b176e888a34752eb1baf4b876","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"bd9b3b7f5ce7275bd4f32fb355609d9038fed94688be48bf618a3c2e0b9ca102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6d281a26d8ac50b58188d06190db9e64800fa423362a34399d5e8db878ac7e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","signature_id":"3112eb82261054cccf2006078302df6fc47e6d756350b2f95e0fd2f5e3283de9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"5e96aa58c18568a25ab9acb5502820d47261ecacba6ffd475cef3d142037f291","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"300e775547bf8a76fdcf7fc914a06bfeb3d60330ccc0c5023680429a57300fc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"67635489ff4055145750e643ed97308c677e61e4b811387b55d6f1c1a7f6d2c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":39.285714285714285,"anchor_occurrences":11,"after_last_push_count":11,"also_in":["STG"],"trend_counts":[0,0,0,4,2,2,2,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxwdj8sw and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-fct27gcqstbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007a08f0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxwdj8sw and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-fct27gcqstbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007a08f0\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxt4mfpn and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-vg2849dzv4lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0004c67b0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxt4mfpn and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-vg2849dzv4lc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0004c67b0\u003e: \n    storage account log verification failed\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","occurred_at":"2026-09-28T02:00:25Z","signature_id":"08f63884b8084a2606a1a5268b6fb65c1cc3e789a692cbd59f3ad65ea31fbc51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"514d013204cdac81f773e4ab4576f5c8edc422d2a1cdb915bd7a4b0bfecf2094","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104028667012190208","occurred_at":"2026-09-27T02:00:24Z","signature_id":"e2c8651eb00e230b2d9dc94cadb3e408c87a5a41e20a9cad8673027e24eb6ca8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"cf4d5b3bdbd5761c2141297e35cf9f5181edb29fb6619d96f30e8fb4d817dea8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103665929874313216","occurred_at":"2026-09-26T02:00:16Z","signature_id":"6222019935691fa885c4fd69945b3adceec721f7a1da3ddc026e43005e19260d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"ab0087d314f58a4462c0a5c4e09f0320ebe8f8af617d9a58fb7a9d27fefcf6e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","signature_id":"3d1829567b78ae4d2efc76eba5be75f4a96678a24f6fcf274e485617885c4edd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"5159cce0088044d8bc192dbe0cedf4bcbebc146393b1f4ddad44f3a9d431c657","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"a0d8bdae7638bd5124c215d98805edbc939d87889da4c9f0c02457aecd6221df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","signature_id":"5c46d71a0a4a0451e999a028441816085c5b759ae39c7276fb744cd76b6b57a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","signature_id":"849d9e776f46a941d1482cb6eef9a594d740471e89a56ba737d225b1ff968668","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":9,"runs_affected":7,"failed_runs":7,"run_impact_percent":25,"anchor_occurrences":9,"after_last_push_count":15,"also_in":["STG"],"trend_counts":[1,1,1,1,1,2,2,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014b70f8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014b70f8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0015174b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0015174b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"e7f9b8ac38554f62beb05357bcb84d1c79a8ef404e325310c520ae6e5285e61d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"1bb95cbda5b42a403cf5850291c1bcd05d17dd2fca6b7e2a40ac7641b5dcb5b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2e693ea3a6a3d55ed57f39150d24a058191d22a6fcfcea4aebe8f0d0bcf9f539","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7a7148bcb274817df0df58cc4f3c17b4d58edcbe850fd8416934c0973dfba2ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"feae159acc1681659551954ca69ac2fec4f445a05e1e2e12837c74f0d7fb1aa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"c3ce37d99ec460bf1b759e73ea22337f6eacbb332a70e22befe3dd9f6c0bf0d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"d8fdb12d282c5253e72136a3836928bc9db7e3e73c7eec5cc70b96bb18cad7d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"b7d9ce39ea6386c93e86493f0c036b5ac8e6406a7e67ba9a7045486fddb68882","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"0cfcc19325c94c6fcd7928e1960873ccb847383b907d9a4008c9f2eef815cedb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":6,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.857142857142858,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["DEV"],"trend_counts":[0,0,0,1,1,0,0,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"f07ab4b774d34743455c8d173177f39c64d2818384324244b685d13e7ea7d3cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"61da179f0b7befc0f6ef4e2c530e9e4e588275b255eb735499eff0b6124e8ce5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8bea3b31627e5d847815834266e4adb45e346c49dee4938fc8669a5a0d148891","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","occurred_at":"2026-09-28T14:19:44Z","signature_id":"31429d7beca55879e992a9e74912d2072a12718cdce9ba4d4ebbfb9d70101e40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"325840a2b42ebb8d501c0b8e4f87b5bd1159c7995917009439b796ba33def4de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ae9cfa1cabd766fc0ba51cde7820c4aff0d9b5fe7146c215e79849948da08c5d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":14.285714285714286,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","signature_id":"29bde2d7b8d5a41edc15f4349385c2d0099553bdf943c64bb8c46da6c996a3e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8de98f59efa15f72a396ca1bd7e2dff75eb6bfdd7ab655e62dff7e62b3c1152d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","occurred_at":"2026-09-28T20:27:16Z","signature_id":"d239f5d1b83b70a7a970eaac57a5e3865875c1f179ca44cad09e2f730112d858","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","signature_id":"0808cac9ab1e6eab75a259dd85739aff3ad9bf80be0b493ce8ad127f37249569","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,0,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"053123c7443d1054083dcab559d2546849eae900c250f58b40fbf47904d636dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"0950bf1fa4b422939bc22e6eec036cdf20bb4658d9c093acba892685503bea17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"9b958b779ca895c4c49a73903b6b72b8677fe49ebfe23b8787341a9e83c6884a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"e4912b2f6d7e7563f8767c16448cd57abe8e92db7831d87c4f122e3d8fc2c77d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4528b434af694f8025e1b1474e4ef626ed5489dcd675669ca845a20091661f31","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["DEV","STG"],"trend_counts":[1,0,0,0,1,0,2,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-j6wfl7rfmftf/cluster-nightly-4-22-nm6bdd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b18ea0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-nm6bdd: failed waiting for cluster=\"cluster-nightly-4-22-nm6bdd\" in resourcegroup=\"rg-nightly-4-22-j6wfl7rfmftf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"name\": \"30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.981848903Z\",\n      \"endTime\": \"2026-09-28T00:26:23.602322446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-j6wfl7rfmftf/cluster-nightly-4-22-nm6bdd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b18ea0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-nm6bdd: failed waiting for cluster=\"cluster-nightly-4-22-nm6bdd\" in resourcegroup=\"rg-nightly-4-22-j6wfl7rfmftf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"name\": \"30dbd006-3693-4ff9-83b2-2e342bb7120d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:15.981848903Z\",\n      \"endTime\": \"2026-09-28T00:26:23.602322446Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-fbjrkv\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000adaa00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-fbjrkv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-fbjrkv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xs76x2g6dgbl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"name\": \"2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:39.891778199Z\",\n      \"endTime\": \"2026-09-28T00:26:43.608164604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-fbjrkv\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000adaa00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-fbjrkv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-fbjrkv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xs76x2g6dgbl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"name\": \"2c0d0d6a-5f02-484a-8b51-9407d1b552aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:39.891778199Z\",\n      \"endTime\": \"2026-09-28T00:26:43.608164604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-7q8vqcfshfst/cluster-nightly-5-1-htcp7l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000acaa60\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-htcp7l: failed waiting for cluster=\"cluster-nightly-5-1-htcp7l\" in resourcegroup=\"rg-nightly-5-1-7q8vqcfshfst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n      \"name\": \"3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T23:32:03.179098884Z\",\n      \"endTime\": \"2026-09-26T00:01:10.002965076Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-7q8vqcfshfst/cluster-nightly-5-1-htcp7l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000acaa60\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-htcp7l: failed waiting for cluster=\"cluster-nightly-5-1-htcp7l\" in resourcegroup=\"rg-nightly-5-1-7q8vqcfshfst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n      \"name\": \"3dbd0e68-03bb-49be-ad53-7a10f8750a36\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T23:32:03.179098884Z\",\n      \"endTime\": \"2026-09-26T00:01:10.002965076Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"c1f77dd01af7df31876c36d16401e499d9a695be8e5fd9fbfafb98309347710a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"f5e23bc64ae36da787717026bf1c86a8e8b33beccc9c6849ac470b476a3e7ddc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103620630963294208","occurred_at":"2026-09-25T23:00:16Z","signature_id":"9d32c182a0e9b653b758b4108d7d2bed1ed38f4bb428003742a75f25756d9bd9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-23T23:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00103f360\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-xqlnpq2wtgvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-xqlnpq2wtgvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00103f360\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-xqlnpq2wtgvl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-xqlnpq2wtgvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2f40037715e35fc92a35242fe15cfd48e9dba77dbef0411ab7466b2e286500e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"7756282e4c2797adc5a9559f0aecc68f5113799a329e53f6fead9f7607a3e463","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["INT"],"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-23T02:00:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","signature_id":"dcdb32a276b85d5762c8f9a4390b250e8e8c154b174fce2166a1284975070064","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102533571620114432","occurred_at":"2026-09-22T23:00:41Z","signature_id":"dcdb32a276b85d5762c8f9a4390b250e8e8c154b174fce2166a1284975070064","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","signature_id":"057d78f20fc782ce9997cb17d11d93f7004a1d7a6ecbd74b197c32efd5c0100d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","signature_id":"b06391e0cd724ab0d438d9c0420d007165f3fe823ffd52497e5d58364ca57ca5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"d33cdf557cf6b0a812334a9c133b09038e1b46fff4e53cc99d59b39fb3a25778","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a7646eda388fcf6391d44cd32bf5781942e0ad533f17bf8d059ce1a303c08693","failure_pattern":"ERROR CODE: InternalServerError; detail code OperationNotAllowed; detail message Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-\u003cid\u003e' since another operation is in progress.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T02:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ff01b0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d8dncw5q6xr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\",\n      \"name\": \"4cccec67-451f-41c1-957a-1453bb0ac735\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:15:20.351974211Z\",\n      \"endTime\": \"2026-09-25T02:34:26.038545442Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-44b92-qd24g\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-44b92-qd24g\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d8dncw5q6xr2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-44b92-qd24g/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-qd24g' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ff01b0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-d8dncw5q6xr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cccec67-451f-41c1-957a-1453bb0ac735\",\n      \"name\": \"4cccec67-451f-41c1-957a-1453bb0ac735\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T02:15:20.351974211Z\",\n      \"endTime\": \"2026-09-25T02:34:26.038545442Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-44b92-qd24g\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-44b92-qd24g\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-d8dncw5q6xr2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-44b92-qd24g/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-44b92-qd24g' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2103303533200674816","occurred_at":"2026-09-25T02:00:14Z","signature_id":"78dffc2ed4c7c4ccb6dfde345d2214bde1baff9b62e1668576e19c12f9366096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1450b085cba9e38b0c42178fa8dd0c598b3bf57ce6c74d9b3dfd6c592b42cfc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tssqn7\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d6f80\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tssqn7: failed waiting for cluster=\"cp-ystream-upgrade-5-0-tssqn7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-s5ql4kptw4ct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"name\": \"8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:40.978774448Z\",\n      \"endTime\": \"2026-09-28T00:26:43.612304644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tssqn7\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d6f80\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tssqn7: failed waiting for cluster=\"cp-ystream-upgrade-5-0-tssqn7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-s5ql4kptw4ct\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"name\": \"8948f23c-1b13-4b8b-b8b8-d745d0f16492\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:57:40.978774448Z\",\n      \"endTime\": \"2026-09-28T00:26:43.612304644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2c4a64bd1771fc9b48285b77345bcc0b66100e0b2b369f902d0ab7f1dc30b0fb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV","INT"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-24T23:38:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n      \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n      \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"a7e14597b86188a7681e872f2938a0cc446f731377f16c3c6f83c2cc272f9397","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4ccd202ac53e63c2d458b9376fd889abcea732d65909fd57568f98cc9601e785","failure_pattern":"ERROR CODE: InvalidRequestContent; detail code Conflict; detail message Cannot request credential while credentials are being revoked; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:267]: failed to request new admin credentials after revocation for cluster \"admin-cred-lifecycle-7t664q\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000847980\u003e: \n    20240610 credential request failed: failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value\",\n        \"target\": \"certificateSigningRequest\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    ; 2026-09-01-preview fallback failed: failed to begin admin credential request for hcpCluster=\"admin-cred-lifecycle-7t664q\" in resourcegroup=\"admin-credential-lifecycle-test-rwn8v4w9qpfg\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:267]: failed to request new admin credentials after revocation for cluster \"admin-cred-lifecycle-7t664q\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000847980\u003e: \n    20240610 credential request failed: failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value\",\n        \"target\": \"certificateSigningRequest\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    ; 2026-09-01-preview fallback failed: failed to begin admin credential request for hcpCluster=\"admin-cred-lifecycle-7t664q\" in resourcegroup=\"admin-credential-lifecycle-test-rwn8v4w9qpfg\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while credentials are being revoked\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rwn8v4w9qpfg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-7t664q\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103983023094501376","occurred_at":"2026-09-26T23:00:17Z","signature_id":"d3a980efa305872f339ab522589dbd2c3e5f4a4a66db733dff8e01be50e9bcda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T17:14:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n    \u003c*errors.StatusError | 0xc00103ab40\u003e: \n    Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n    \u003c*errors.StatusError | 0xc00103ab40\u003e: \n    Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","signature_id":"0b0108efd360078eb545d9c8493397d07692133492d39ba4345c2659c856781d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.5714285714285716,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104345445676355584","occurred_at":"2026-09-27T23:00:25Z","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":33,"failed_runs":31,"raw_occurrences":867,"matched_occurrences":867,"runs_affected":31},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":485,"runs_affected":12,"failed_runs":12,"run_impact_percent":36.36363636363637,"anchor_occurrences":485,"after_last_push_count":861,"trend_counts":[108,107,200,178,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-24T13:55:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":7},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000994380\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000994380\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"02065b62056fdb1be37e1f143fa9e3989f4c00365df7604d5fc5f67b38deeb74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"07f72b351d9885024483bcdda053ed042e3ef275bc57e4fe920a64d613ecaef3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"0886a5c6a659b6005a58f499f6c69484cfe98d4e6289c7f072c5c6904bf7a54b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"08a625905fbc7734eab24c2da8e1f2f3c7e204001ec83f60fdbbd150f3f95d10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"0948e3f0cfdd80324d14616eaa43a0f4cb0a32ab76564cb78669d9d2dcf1f472","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"1cec2a79be4975a9653bfd8882733838f7e123313a78520fb3323f7cd2a476f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"1d4510859afcc6e6a951c71249a3c54a4d1b3907f437de943b3dbcb9f98099bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"2491fcd2eac3fe129ef5af4e2165dbf276bb34dd6d7b915bcd09f12fdd5df0f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"2496e95a86576d0de268af2d478e259fba5936be455d300c7e3b637d4c9c0436","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"25a79014eed6d04b3f486b4bd113cebd8ddda98d4daec17f489c251260e84d81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"260a8c3ed58b404fe205841beee67492d697ec8ff77258801ee95054dcda2df3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"28f63e4ea877a5226e030abfc09e558054018cbba5fc549e1976e79199d95ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"344978cc9fbf73459cd931f815705d7f83f04f19bb1486d43bed4ef863a846f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"36eeb6a1e1b48cf8e706db53d0be8d91860b5935eb3db33b0ac89092c3fe1e98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"3b2acae45c01d2462c31e39a66ceda7d89be3fb818bc996fd0c77eb9f0f34519","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"40a33c47440e8587f6d078af5f5c16c35ffdc39ca033c2eee6186595c1a87db3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4825f3fea6b1d5d7b75547790c7d34a8a40c3c76a119b4bc51a6e1c1b00426cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4c79d1369b373f888a3b45bc9980f8607e90b91cf48ef5a98e4523e05cd34677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4de0b73b423ac48667ae452b214274712219be45eefe629403a4f7ca9649d73b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"59208ccacb71632d7b6b4858b15d2c7fc131d39ca9231040ecd620b898b8bc2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"5fa3292ca69ac2b8f8914d64f6c6e71c41533069457c029bc02a6af50ebe3d12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"60ecda7dfb77cb74a1b276ceee25e1a751452462d2eb558d731953e82366126d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6551a9274a31031d7c246caf9f38b3803753d5519917aa5228c3ef2caa0fdd99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6a1c116e54ff19de705618b83b20d85085d2d56b50e18f8214b3986e60f160cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6cc2e47633c842a292f67376ce88252879c7a32f0fa0c8b61c35f7e7b1a49ce3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"720afddf7bab9bd3106da1f01488f44f6c8d6565ba8f839cfcc4eb64d06a9278","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"7c07a109a7f8eeb6001f12064984f5de864be533ff4a294d3a61fa4068710abb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"853ae8067a45c4c5af2c0b037c1dc51d90a61b2a9935e489015743fa89a40ce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"92f971cf89077bcf6f7c95c75f6548c5f63e238aa46a1aae576b8a17cd5e41f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"967b4f667dc4e0c254b60f79b36f69e3e0cba4279cc5c924e4597a63565d1a7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"967f8bf89265c03055c4a80ded5359ab4d74d96b7b327c590c07ba6380770600","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"a098cb5165f51c38edade056cc88516c0f0a3f03634689a03285045f95820440","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"a742fb03b3de74be1d0b58aad6a2950c30af0f017f641245d24c851cefe749af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"ac2713765a7efcac04a7835d38ada2e943f7084564106cb6c2933a048a1e3802","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"af40d913adbdce8e6103e8528992e78d1b771a4afc5a2f9de4c2c5f4a5305fe0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"bdf59d33ef572133b373eccc869cfcbe6143231f3e4511de63e7d2a518123c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"befcba4a92c1063abce85eaebfd5f49c658485ca00a56047126b556f822a3702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c243f446d5b36205395c8a9d49707a8bdd732debf06f3c70362d814c54919a5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c356e51b9b454f478643d1102b07e3cc98f9d26f83f8834c798c0a3d0e6db18e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c6bd5ae45fe994223d884c6fd019d9bbc980c09700d74c7c6ec6d11950bf3641","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c70f585d6ebd874ebdf87f09a6f5335bbd8ffbbe2510b6afa7a3cc5775812ae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"d1479bc8ab6f8ba7adcc3af3628e73e527d7fb7d97afcf0aa9f15d21d77cfba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"da45a36375afba2b38843fbb63997abffdb70fe444c87c2a54f106bc6f210096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"dca63c35bedaaf7995439199f45e3ba19c7e03db5fa96015f6372c55929ccc8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e1d895a054a411be98f3dd14706c4b6811158995094d211b7e4fe2cf75726156","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e2176f9fd669f072305456d519992c756a61ee3a64949c662685d7e2dec03a2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e4898a44c2d0dd7c9bad670935c424400fc52e32d0af7e2fff6aa931d5cdaf60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"eaed15a6a2f8b355906a22e9b7c33ce97a5593a10c5d7448cb923809e315d678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f0a12fd2d92b9a02183a192dbf54b969f015ef75889c772832ec9cf6d3be6b18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f0d90df73e0ad52f9f8a704c64e5d1c3c05332de051b7b56f846352299f4f2e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f44007f9ca5ba215de70c688ccbfb0a29ff777eb5bd506af4f556c69185fea90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fa4f63d6822358edacc49b065a20427c02d8e8a557b09adc67f44ddfbe510a96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fa74be0ea5ab5dc482d53b08882b7452407715e934f79592c346c3604704b150","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fc7b68991da4719c194ace05c0d2d561215021a3d370ce936079026d3369b37e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"ff83937b404baac989bbd19b80b0cf998ad2d8f2250a11efd067cd3656f1c535","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"0de87cbcf5c9877b7a4237a0f07ba9b9a2ebc8fcc8d43dd86eb431dfe3ce5bfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"11b0056d5f9eb91a6078381a70f2fad5d7118409110b509dab70404db8f3a474","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"146147e4c288d084c17ebda4302106fa26b576d46636bfe7aac69b5bd259555c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"19bb3f1582062678b47c57215798e7ed69eae547e6905ca14e129c211f2fd95f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"1c03a99aa561093294de0b7f438653c656628e4496548c3d82bdad67cc2948ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"1f713ffd36c3707139930bf34758bcbcddc571154f3158c1cdc2ef4e6eb94f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"33a53d9e10e046f64807c0665d9a96bdf32e483709eca3a34cf7d6405c07a06a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"390f99fccf43a9fdffa950872ef3818be3f84574f56671796e24f00762856e39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"3acb3552748b9fd13e54ddeaf5b07bebbcfe389a5c337c06b8aca75747b6ddba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"3da6d511cfdfcc9a966ff17298c613d692f9d8b2b2ff41d8fa714b027d68841e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"43e7f1565c600241e1717dd21fe3ce831088fb47b5bcc65de9a0851b50e2a95d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"4586bf9af4491d16af0f6a47fa3ff73e6e59cc51a18f4f7bc5b6a0748206dfd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"46731606f59a9c0624f9396b197db65e833127d4c0d807797d4eb23355d2fef5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"46f505820c3047e75eaa0fdee320e69bd3f3b2fb1616f3230268704c75a50fa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"48577c445231793eaba83204969b16f58df261f371ff2fcff7713d00ec5ff487","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5a72b9f3b17f460b30a9c29aa4a08aa0bffaf0211205623d232115f9e61c81d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5aeade5b036df850b438dd9d19f71214c80198c23e9c7a18159685d5d6127efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5bf9671d8b88883fbca8a3384f2c419073f4736cf8d87ba15eda4718b7e70b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"60061ef754525bd1335a2775069174d5f92f29de7213ad5df21582aa86897763","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6708cf6358c16ce63138d022c53fea4bcf335fd1b118e1a4b673daa90384d30b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6aedc299554b7d252bc79ff70f6fb948566ab1c2514ae26ec861e15fb35fabb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6c26b8862baf61fb815a9bcc378adad747c42890538b5734c6866f9b99fd3e4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6d40f168b82b944aab38b5963703fbbf90d9e0d6a85bc07a462e258b20489400","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6fd0b24e06192fccaa00e3b1c3b94dc37d332c9b2cef1818ca04f0a13ee89195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"78a9fc61dc31966c8f5382a6adefb305e57ed7ce933c7353322e5f5babeac4ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"7beda16600f383a39e0d3cc113f1193f43bcc9b23fc90352dae1caee67106339","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"7c6822e43843c1ef4b10f112f34e70d1c8fbbbade14e202dbf3988631790bee2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8389b7d6027c2f391604e97713773076785e864f55a45ec1f88e9f79114dda44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"86cd7d2264f1df006b68597c7229c5b754c8bda3b5ed006204597c09ea46185d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8d7cf3325e3f51dc9dd535c7d9b929725b10a46074af389b09950cf2fb1da226","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8ec6de64c81fb8d075d11e6ffaa54baf82b7ba35b1232f6b902d46f2c56ae52f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"9276c54a5fbffccb202a50473f6fe048ba2a9c847ec78aa22b393009137283eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"a6d19019536f0ffdf7b8d102ecd1f15f3a7e0e6fbe23d1a367e4af2586c60298","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"a7962d5682a436d220883c1d5f6842915a2a574c423b72fe7acf314cec931af4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ad0100265512dd64b07c8991367895c07931a266bb11587312fb99858cd39dd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ad63ed6b9b2818b36afda8be9a47960e810863d76780c0fffb31e4de99ffdc65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"af55c67ce41cf5118af08bb66b3fe1ad5ba797a697ead874cb5b7babd4f9361e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b107cfaa0d2663af2e4815be747bec7cac7f76a98c014b790eba42d70dd84d23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b24096ca5ffba57fc98933a2488e339ea00b6584d3690b683f4ba35aeac04579","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b3c2fd359c06e85b6c30642e7db5c4f3164f911439b26a9ebe32dbbc216aba1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b69976e1243e6773fa8a60a6bf57877ebfa4d1090f0f4bf4996c7fd78024e054","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"bc73ce6f20fa67a9ec8483aefda2a2bd30ac78637b62659bb7edd381600f8d94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"bf050428c36d99a435724d4cc497547088fd3c8b62f9ae651e02fb9c333b72cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c2585ec4226888f6f576e96e437f1b2775fc3ebb2fc2462aa71665f8f2c6de32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c28773c54c9830f7b9e29005190ee26d6777b293c6a190ba25bd70d0a8a59efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c32d7fdb1bc89858a63402558b43f56a1300afac08a052f62e1c545d91b7df40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"d173739cb5ea2347d8714d542da4da098196c8a0c7005c4787d2d2146a3dfab0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dc40facbe59320069107fd1a5edbe83f0018e18aca83e4dbb8e3245a4a984b31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dcd5e0fbecd4cda0aa2fa67497226a4350e6689b2cc3138bafbad6f78f1ff4e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dea6b577a82026d7021eb80143d812f0e84b8b2c556ecfee6f5490a8a7dcd1a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"e70b708fe9e564e7f0383d82d6551c4cd363a16cb4041d76a97df34265bce1bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ec8594b18716573125d007b01c1d0d6dddb69ebe6a0ac57f7d8b0911bc05b4d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f38f22878ff4dc7e201880848f8c0291b74fc749dbdcde36131160cad489c060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f46adce6b1993d0da3f46389c99620d37df456802b56bd77fce4645c6ff1537e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f55bcf54c5d67b37e04f8a536611e03e87fb64780bc1a8305f99107bf6029933","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f750b8949c6048df3f06cc60730c23f17eed1205b46d7b1bef29f0143eec2ad1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"fda77eca2b5ca7a3f170a1eb6c18547b563d0dfccae707a6948d12a1c6e551f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ff4468a0a3f2435884308ecd6b5b5e9484371fe0029dd446ac35d4ff6a062092","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"015a62d796a14aafc93ac8fd9fcd0192bc1f4daaf775afe7d5216b4fbed2d422","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"028ea5601862fa590d5e2201ae315bf0b1513b6769b08ed07c16e2de0ae99250","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"0567435825b2f1f46e7c8c0c92ccdf5c233255559fbb4ecd19ff369114130a56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"08753c85308ae68cb595376a3e6a02505c40bc1c3f231edf2037c7c84ce54390","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"0b322092770ddc32a8a6514dee884547809162ca71778488279ef0d524e10934","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"1088ef782bb08563b7f5eedd4e1fa2ad693b60b2a406cedcb3466c7edeefd7d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"14576de538e43a898ea0afaebbf5578a0ad679c60e4b538e2c28a0d08595fffb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"195e663d42f52c6c17db4a06e43816ce2ca15b96d6fe36189b86067a43dc4c68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"1ad44de4f84225657a0f5171167d0ef345449d251f077c5d71045f9eb0644d77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"20d5cc742c1cee73f1b04cd10f6ea72323a6cb2809aa9f9667747252b4035ef5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"29ca14863816cf66991a82f9b46c9729fedc201d00186088b1efce79233b37e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"30cacbef53a9519a12502d8baa9dbf2e4ec518774dc9498d9b70b0e3f05f5dfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"3c66cf27a6f188c03e99e0e620fe8a8673cd9acbed4dbb1db198db2a90216cd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"3f517ece69506cbfef521fda194e3fbb39f7991afddffc45f05e2fdb1b402cc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"423808405226253bf2780a731584c5c00ca80e8ac8b89cbe298d4b9bfaa20ef2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"42dab5794581704633e8583d39e306fe7eaa69761c0b3bdde3ae6b578beaad85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"457829d398eaa8002f3a2d02b33f53ab433c3ee054e691177cd91348662c5085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"48d0bbbeff9ebd85c0d242660d3ec069eb2a39489e5e738e5a0af642de48e47a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"4f8bced1b86c8983e52fabef07413b07a835bc6c67a4fb7b1ec887b5f4487e37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"5343a8025b83b6578ac81a81ccd0c2c8f82d4d10cd53595fab1fde76e7b1139c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"587a41d8cddc604041bb53b0856b1237afdb02c7190fbbfe6c6a0aad7d417448","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"63c131654901b385f7d641263b5a6f2bdaa7a6105cd00a04562f380454869986","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6a3887e150d79b2470eac482314e5825c603dbdf1cd847403c74e9971b98c5f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6b345d5ab6014d402a2d652c0c5909b61c6a4242f0cc2ffe16076dc13b9b7b17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6eb87e754a0c8e7de25c379f966fd9286c4d705599dac33f311b2d71c14b420b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6fe4ceadf290c9cd17620e6045688aae8219b7ba271ace3ac6aa8d59bd4dc942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"70fa408a82d01d5e0229300a03628a7b7c697abcb7a4b9d83bddeb0c17eeaedd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"75547c088779acedcc0f688ec015777e98c9c620b93369dbad684de8c96d5068","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"87764a3a1c887346ee5ef8611cc6a8377a04492e4975331324104e456976820e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8a7efe49c03007d1fdf090e4785ee39ae4747a5c5e30553be13dcc9aafd36922","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8d03287f7ed3f0349f7714b3bb9f7d51bb8a6961412a9db10097515ba650bd89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8d9b406fbbb8cea6732da0cd609f796ba30974cc05d728b0633597b50f8c0abc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8f3b0599281e22f58c3defe5f0d147b488c1b6a0f16143880bf51ef7355d85d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"91bfbb27fa9e016442e0811e84d9a5a5a7c3beeb9aefb272528d99cbbfde1693","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"945c3b0b90dd70635719bf5c4663085b88eaf4bfe2ed27a8f72db49c7e4b63c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"95d33091b2e3ed0ead5eb1ceae0e4c3fd0d888e8ace45c0efcb0c25533a58f9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"98bfd3289c6b69608510ac8f99b14ab0bffa05824f201bcb73f80da461684588","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"9c4d03f3479a5c70388413a70424168b5e2b17ad421c1bf6766436364b979d1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"a85e78a9cdf1147b1107f8d145242f52a53fb32fc15b96f67ddb5c9c38894042","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"abc370c7132d3dd0cad4c01b1df33504e2a5ee15cd985778c8ebeb08c7378217","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"b9ad954e8151aa5b0826d0d74bf5d49508ba660587f9fe7ab54350a7e715d196","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"be867ed293f062a9bd769325496bdaf3731f033dfbaf096811cf5f767ffb8479","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"c37bda4304af7e53e58a3919e84b1477a3a46695576de8ff29d4640d43029591","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ca41ca40c2a7ea09a51dc74552627e7f098ec66a21f33ac2a37277b8e785ba2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"cbb84a2251ba6846685f78b090898c0a3ac4b5ed2f079675620d0a5b6d6c16e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ce49b331dea25dcc6be08ddaf7d9a319f6c26e31fbd36c370bcef547c757521b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"cf518dbc290a9e0538e3f3c154869ccf678a1dcbe5e6134efa2facaa72d7ed91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"d8cfcee194e8bef7aebd6061716b0c8a42cdcab9130e4a23056c9e64bf846076","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"de2fd51098d4fccf5d1bb093ad14ac97bee13c527a30b84d2038a84112fb873e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e205db983efa4c7fe30a3098f99c7b28db976345b998b688245438d48a09e9bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e259ff9177d1f12706b36fd539e24cc7a728c76256da9b5c37b773690fcf29fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e5cd62f25356431728e737caac9f39e8423781573072667b098d01ae01dd194b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e9a3aecc6dfb1296c32f694be25ac887dd48af1f7379a8a0c02b86cd85efca8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ee72d14d762ba3e9d0e27f0a7cef6b5bdcbf2c73ce4814de75d5572f0890fae6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f8ae1811368ec12293281df96c6cee170ccc9b5c6f2f7f431f5e0b46e83d6d68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f8f1d68ea61706af805ffd1d510750baa305613520982e0bba79e29979c78292","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f9c44b3e72b1127fae5b53f6d7a080bc8f4bc6b26d9f291fb7dba9911f6078c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"fac67d5451426d0f28252a0fbe031705ee4850720c8d46822b1614d3f7cd79da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"087c5753d272a2ff86285ff6f45823005418b1e6f89d4b2bb703f0d6deb2094c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fa62e44b14b31ae7655734e567ec10b60e7231167b3936203b6ca0bfe48dd179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4c47478a4a085b6edffed2a7105111b2464ffaf9972aae8beea63b50843450d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"52b1633f73d41b43932bc93f25195e310e768881694919964ac5f253eaea7842","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"1353cd7fc2e7eb95af2a2957b1ec2d2b9416cd1f12ca6cfeab46ea5f033a2da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"3c01497d2687bfb43b9288af1b54486f516b205e9813d8dce28ff93101312372","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8b9341db534e17f1c007a11f07836dbec6a14166f62a28d78b3b65efe63e43a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"0b2e0e2a705eef2a8c34a9cbe9058b556f7e534770a547f7b0609260c02c4bfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"0d143ad9d871f8761c894631d5ac5ac89f118ac56bd392b9fad3aef7ef29f7cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"103c847c3dc0e0c5a0c1e2041083d44a521ad4942917fb41297142b8bf729ae1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"18a080cede1bb3983c6e834e6eebef5703de52cf674ed437a488782e09deeba7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2789ba3cd7312b4b858ed6732db313a63af15c5443bb24ef97632db2fab0aa1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2d2fb06c8452a100942b711accf2add79c08e2ad05176213f3e8e83c23c10eb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"3c6d3bbf620206422ebf37e0988112e383deeab2de78cc2c3d71820ae7402c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"44477526f3f5c967017a65f3a9a523c776dd67b3ac4e6dd0403372636e3b1343","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"4c9e4e595392b76de5e57905da765e092ea4b53fb4e4a74a9a478b70b2a59a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5783e4c8b0c913d9ac1cfb0f45eb24edd6329f5cb3aaa18343f050038c5e6d71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5853cc43cf7425055621ffb2ed66038b5546de23d7a240578f0080ea258e2b72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5ab72705e757a4285a503be51374c0f65a83934561314f0ab6a40725e9e8423f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"62b8ed0f3c7495d4a962b220108f6cdf7916768f686fc9f71620743b5266548f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"66bb238f691dab83756a60fd4ef63b42d1bcad07037abbae7bfe4492035f5ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"6b6231f6a741045d0d4adc8542106388a4942ccd90466630d8b1d4ae0197568e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"760124e7b25aec0cd4af5e7f97d46af297b8834a0ae47c0dcfff744bd3a49ba5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"76eeb452365a9197cccddb41b3c6454114a1f922f7775ab7586fdbf3211fc438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"7980065f3e3c71ff5248c023d7dea8b862c74b6f0fc9d85d1b0dc6232900a78d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"848a835912eb35c98e5793402aff5be6e85e6150ba7567ff6939266d31765a3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"8ec3e712fd3e8543d2d9d2a3f6c13734045c633652dd3994ffae220c9f4487ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"9b3fe187d08170fbc37adc61518c7eeb7b25a95802571b8983f126cabf6fdbc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"9f4ef32027d9d349f278002e9bcde48f6415631082b03813d234b189e64d7959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a3128fe89c2b34a47c935b73ccf2c1a9a2740878a1a3290746b393ecbb354616","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a5caf3ca78309008f43557f4e7fa4ff03bf651fed81afe60cc0f43daae52ccfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b06bc75230ebcfa423f62726ee3b474f1b5c932e5eb57e3f75869553b569f537","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b3402bd39c1526399877b423af2058f9c4707b63ec0c5f879b3458d0c4b9860c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b7d497a59299e6705ec5c27d0954476df3b73d478d675970b241ac6bdeb0beb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"baab246acfa194162e561c3c3cb6c1341eae11f128345f24470a2c72bd2f4aee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d4bdde14f6cc55c0354fc8f6e82a55421657445ff0aba3cf3df52d2f44eda51b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d7884279496d93385c3451a48331a1291d1cbd9cd69dab41dcdaacfdd026285a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"df16ba4a6674c679f09b89c1eddcb51a1f9ca845693dcf206c2902823ef7e013","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"df4451d3e29bdc3a64d7d56a21b5e669cdcab64a57c00193734b831648f9892a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"e403149dc0ab41b7958114f780c8dfec643d355ef165e66823690ee143d7dcbb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"f3e5b048e00e5607edfcc1021209c6ad811e1b8fec6fd4f78740242a2b4ef145","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"055fc0966640e70d474c53a699b885b63dd8c74064a9e17fb602a8300375a12f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"0cccb5c6694b8a6067983c3b7bf775d7a351d03d11aaae0408425a37126b77b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"15fc45cd3a6e9b691b043a24977de024edb4d4590b2e83405ee977a707b3bfbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"177d3a6c2c33493e767ecaca9b49e225e0b353fc1f73f3a7f19b8d1d42de2b31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1c2d92e2bedb33fe555fec70451124e32fdf767ba423de0c8f828af1bc35cc8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1dd2c6cc8e4f394a34e0fc7e8213f2ab03bd15a43e139005281c4de11c90ccc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1e11434627c662dbbe902599e9237bb9f801d7b594c61e3714ab1bc17eca9161","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"1f20143686241c78e93d2d29986f7163132c10ed802ef6e532c64a395fa4d866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2216802953356f2ab624815ca80f12ca32ed7a86476d153301b4ff00b4a54b22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"22d721cd87f00c16c308d65485d8483188769f0546a555d083b414ba8263727e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"29c1e995b8db1156d89e071660ed02b25db28529121ba2f957571744517c5a69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"2d025d742d960f65ce0df017beb150ac204ad79db74000677aa6ae4a6d667007","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"310fff41d54de3202e5f9f4c3155139ebbac3cbc401622dae98eb10133d7a4af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"31b6db9373de5fd907ddd06fb828d6f181f6a35afd6c454485818fc463598c5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"378fe00b23b03f756b8692d9b32ec7d2be8def98bef3a4def31f682ef75a69e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"3fc7356f7fe16e5e609d0ba5038de5a411cf304f107f57ef9c728c54c87f3726","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"4128486c577bef73cb4bbdeb9e75cb7a1aa87e84c86bf43283309add2a1f3286","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"41eb5f994f3ddf34344ce72d61df3f563d8898b4a6450b02169b782482eea506","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"422f4827576f5663abf88da2acea909e13a8a93903be9e159726618aafddf3df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"460443b8e000eb1965f80fb4652248f0dd60ecdb22f5297b96d6782ac13c0d1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"47aecd3202086479a3f88dfa9da71e860867c259c29f1b085fdff23422a0e308","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"50239d603bb780e2e003b8ea21c9046e3218d0cfe34261f655d2951ccb605eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"6734cdcb4d8aef16e200ba287bf768f9fdd46715bf7a03214569d57b945db659","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"67ea86669999abbb4487fef31f427939e9e272ebfccd1ab225343a4011bef471","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"6f1244f44ecd8b4386870618b25b694425b1ab47b331ebc35c4fe4bd67d4a631","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"7921192492632722db01f265f3803c9c63f3df67888105c25cde9c26f19b4bb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"837390542bdc443991e02ac2fc910c99edc2dbc38f20395d7af528a232fc43e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"85546bd52d8389ea58add506b472388cf04ba0e5467d3dc4d5f9a09d41818886","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"86320aedbaa0e91f0fa200e2296801441f3c0f977b35ca4e9212e9d6e4951e15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"88fdaade50cf0d1f17b365e4c98e2ce7167b37da529946fb1b6ce451043f8534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"8ebb44433de75ae1a59b237d32db8b6690cca6192875ac16af87e54b2f413ceb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"93293e6b4fe69ff746741e539bee57e0ae0499574cb09083e0dab652c3f3b297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9c9ed5a78aed1b362d5aeec8761fb5cebdedeafacf061840b8e80db9cf7b0faf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"9f7fbf5b4c12c9689dde71f7903041c031a8c2c9d7c2349fffe2cf45bc4705b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a068ba3d18cd23c9aba119652965396d00f35a6fa23cfaacf2c15e2793881de7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"a597ae1c8f906eafb58886f6816c5618cf09b5cf8ed7636f9cb0360f165e8bed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ab4cbe036556b338289f7725734bbb513d944dccdc26d6efeb46fec80dfa5a8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"b64a093bb316ed23148a238480d3c3b297377eec6a0d5b6477f4d2ab1d28f450","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"bd7a905f012148451aac90fdee9e8bce962b34e00b90af1ab067a26e5e89f381","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"be9719f4635bc456af0045a5b461c223ac5096f83cf7ae4dfc406fd8d8ea51ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c623dc307c56fc7aabf82a0862f91773eb11d3a0736caf6f633dfbc71a33cbd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"c70dc9b2344a22c84855927d709917851252038a14a65b1b42ef8df388ce4408","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d4353e2de7ecccab93779b4bcfa87a57824ecdd3130b162614c71c910af28701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d9b4da1e3652b9dbb42131926eb40a3f68dc2918cb70eb3c61df45ff7270e18f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"d9d382c4357a2f83dd034a43e9ce78fdc4093d1bcef932b3734fb4e2aa054e47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e0c1ab0a923b3315a04331d9fd43f31fc3b87e0ba01d6d856f153ac9ab08804f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e6c51f65ab36666b3dcc2d0da28446428775135469fa57e5d425afa8e424a2a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"e707ce73c3444549cb1b45692d1a7984bcddc94e2b321c8ef6f32f988e83b46a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ed35adaa8e06ba709899bb9363d50812ad914c59c4485a7dedf2f384e127f581","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ee1260742cfd6d8b1b7904c0717f61165015ad94e818f219111800249f781b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"ef11ab2cf959392657372fa0909fc2c548315a7d685b35f76171056b4abcc194","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","signature_id":"fd4fd6cc3f3b5dcc77f5c5745a3117eaf3d30f9f2b15b40a36c3cae3ffdee395","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"03b6a7c43565f4d6457ac2a12a167a7c246d083a848784af01b179d01a1b1959","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"08271482dccdc39bddd2cb376b98b288f0d159b3c06e543fde79c1ed7f9713d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"0991f03a2ec81a59452280bcd4e0f4b5e4ae53001ecb34846994d6f30417c74b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"10750c53b0c1dc3c380bfb98ac1ab6ed054bc9fb7563fe02eae47c73ddc7789a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"112783ba7b95c403dd3d93efc5449c10448c7bec0d1079842d6a9d439cbc44d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"11d8be1ddc0e0a10220e86111161442393e8e4fe9d7bc0f56690967982734e25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"1861c336e850ec720d54ed12538396a5968d46ac38640e35bbd2462f646633a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"25c4e000866ba7943d0ff38f96d5187f3e10e9481cdf357cce05365f644f12c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"25fa332b34d3d0c6a3770683fdb0f70e3a425bd1ddea7417e681b87d353bf917","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"274d98c050c02ba40012737f76ea63998befa2394a42cec617bc7a856ad7682a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2d9c7e2f49c360d0ece72f03034d96b551f5b6fc5126e16052ad10f272e652b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2e22a4a575095366d9288929a006514acfaf8691e9e3d7d3e79cb23cd3ab9b93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"2e7c66dd5b354188df699a79024b9c9df5f0658bb90ca1867f9faee1ea20cf9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"30a0cd932f16b4934c6c04f40a2551a1a8dfd6f1c7e9b6c8a18ef85340a6c9fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"319cb259eb78c0d1bf8cdf9d60b591524a4d613b7fa239d634ac6dac397f50f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"4983315498d70cb31e76f7699a313650db35ece01cd779a279712dcb8442887b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"4ae18b7f4aa0dc6f2688473720e3fac39e9593abc7a2693fad640b4439398de5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"53e43e1b5eb0e6100a4da856364350993487635e225b11af657e0f6c38671ad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"56aa8b7d41bd143dfe4120e95fd6d7e4c9bbd66a4330de0e627de46d6435247b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"681de8d8b7da047de7080e050ef511cddcd03a883076ac9fc82e103e6deffdcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"6c40a656c38a297b57950efbb351560b8611e7673ba986f528cd6c276410f7a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"743a5ab4bbb37dae80df81db4c44a32cb21bb079217ca716a1aef7fb333ab9c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"74c7a39324f13af264582228997f17104143bff25f38bbc535fb0f57457f5572","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"75ef17175d53859e312f86a0739d2f6b44c2045f9da4b4c135abca826451b8ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"77100504840186e9b5fee4b7540e7daea129e844b89dc8b99d371650c4f68531","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"860aff23781cffe752cf788549eb40de12d922321ece7a067ab30debae897099","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"8637b2f6ba5650241aaaaa116a2282cef63f77b6b74df2a63cb50451ef5af63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"90e2f9b7aba95bedd6261d87467024ea908cc72b001a451c8f70266a78eb279a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"91d6b2290275c146a985f2956ee1842c1adbe4cb3327ef55f9f3f2426b0667d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"933c56da08d65cda2aa6845c5dc6f3d13c9baf8e9bf94638891a75c18cc282db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"96b255a8cd6cf44fdebf8d73507be6fe889f89f698f8b5be8e30089737a7dc45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"97788927dec662d1d0e17e084bc25e4afd04ae188871b50b8331a02a8e1512be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"9ebaebedfaf6975ff95a4970a8f1bff72b03b8677d3a8aed73410a5bdba9d566","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"a1b4c80ff80950b4b4b75a12d415c5927a2f10d62f1b79a4d8901849717eafe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"a70aa82f421e4acdb041ad16496c250af687badd0c3ab467c9c3a946de950fc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"aaf98359994aa5a99a180a2d9a07c8e60eeb104fc6de4939bffabcaed04533c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"acba9062b228abb646f0453b339381dd60f3095dcd2638eec0023c8404f155f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"b5cc2d0a8fce4dd78b8efcb7b33bfb25bfc5ca2e9fa45fefeb729b74cc3d9a91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"b6e080a5fbaf018893bc01e662e627acbbdae0f7292faf237f307e3db8105b47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"bb36fc8d69b4ccfcf89bcd6519a5915b66e79f46ab6877afd7421eb5b01710fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"bdb9550e9115080d784a804bd1ded215f1f23126748ec7f4f9a2e0e56e195c67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c13d02c7d475af09cbae9d547b5a17ff9551e2bd6740d79a99ba3a7e492e2feb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c207cf1d189d0d3cf54778aacedacfba7248221d8883dbb57f709247ee506b89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c4cbde7cb1452b4cf73b0cd9d594e432b97d6987d73cde5510ab0cc87df2fec5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"c89b3ce1fd98fb962801441940537ea6f73edd87b76bd04625e9c99544707285","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"ca6b9a69cdcb8d678a64cc522c2eded46cc7048e98db8b60701be431f4451559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"cab56619bdf677848c93b5b8420ef2c26d700e1764280f4910b1d4ffa64fd926","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"cc7a5cd5ee8206e6d20ce4a924f1368feeab1325081fef357d5c935b3b9d6af4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d00bb74f4e653fe5a51ef79bdc35230c19f456d7de25ee574b5724ead42ab6f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d2bb6e2d403656c49ab04e83c3bb7e2ff4b986e11e9b4058682e200cf346b8b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"d2c0b34854603aac3e86bd85677fb864f529bab16a93499e7d50cd286c96374b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"deaac275b010033e9f5d3593bc261037d301f4d2a1bc4467c6ed5a8265146456","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e52bb7c8b8702e4f5f0f12d35371440fa26f8e7305b750f17f3780409d1cea4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e7aa44e9bbee4b315fd192b4e7f53f081c938e5e5502ae7ad2f73d753bd1addc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"e913588f2e94836d7f113c48b7e55eb49e70db7bd51ad09a6a1f974c6f20970f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"f036658b7e69c80c317d2d24a945358af0e8f394a6c34d18d961e843159d8f04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"f986660f71fd5e2eff4c2a2d31f4185e824ad6773c3a62840cd61cdbf74954e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","signature_id":"fc9cf6cd8f627b96267ce3a2d2df20ee00f7becd9a6bfddebea3bba60248db33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"05e00ea7e8fca29e91df7921854d0045d7d268a32fd5b1fb898fc496d1176180","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"11d5639b580236cb0256012436dce0eb857fee45e4e28bbc667bf91a1e2a0356","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"14a5ec07af57bb86a839a52a1b638f1fd866b70c11189aedcf0432ac10bfe09b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"16a3295488b90d396bf7321da82f4d92b00ef4b32d39dce5c38e9923b96dbc4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"17df3d13f5564cb9f511516ded4286c659befbe31426e6104494fef23a28c7dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"2151fe66fba30fc7b88c2c51c44e694bfe080433a2c89a3e8c0231bf9e722eaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"23635ac418e137f892faf70ba7be1206ece8952522444085c8aaa3f5478c3c9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"2404a21cae14e1faee7ca7136c2409c3fd7e7c4f15c1dc45b8b5f77f87c756de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"289eec3a6e696ef46ddf1c7f86dfde24b915efbf8c0d349295b1e0b4238e5bb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"293200cbe5510f7bd2d0f01fa85b6491a31df5a39073a66ae7eb7c5c4fada6ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"39ce6c36b21ff03c35642644eded4e8649bd345881f2acc5adac6b656f377e15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"3e647150f33e3ffc56d2c1571c807f82cff11cc470178b3156913149e09c5fa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"480cbc0b92eb45b732e39ec5305ca8fd33c7a9199ad3f175f39e7864760ec4fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"4af054ec58b34153ea3601f9333faa1564f06c8e2e087fdd7e3bd3772cfac636","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"4cc6e4155ae2365f53ebd0b1ffad135c6f6a236f7ef22d5a24dbc10df7bdd269","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5364033772494115e4b5f5e40cf01432156a12589305922ac4726f437c03dae2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"55745c23236c64509e7a4aa08f812f9a02fb58ccdc1c1d4ecfbcb51cf8a27003","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"56826cf4f16e45e7487b87b2af3d3c6ae3a0e3cfa1f3cc7fa5babe12c95f8be4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5a6930dc2b2b119484f09428ee05bc3e9aecd5bc4212a7db9f2b1c6cb97736a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5afa96be6ec41f79a3fe07b706176526c35b8138b5810b57ec811f7943de2343","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5c4cc073f23d1d65b6648dcd7501f81e8e79d5adb2e4adf87d81a8923986aac3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"5c703ec8433f62e4f8db39dbfd922f3949682c8caf1f3cf28f93dfdf421976dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"60b4f8f14ba59be897db2077b4c610fcc0496c981276a321ae544895b832ed97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"61db7bb228bffe819303eb1c357004a6e589362fbc7f05ce28fb3a3517ae14d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"70e94d44bbd3c7bd987ab03782491c93ccc9a29d5f05ba7e1fe547fb16dfc8c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"73468874c7c9cf8732ad731b3ea78e3133f91cd177d1caf52e6b177a524605a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"7a99d6719a1f3f880e142b01cf1409fe5b22d96c4ae9d411a7727111fa56ca3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8790ec9b27ab1f793c595a203edf5c900eb4a06cb5959f9f40e481ce25b3a61a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"897c8759b2596cfd877d3de8c8f27dffa79f5a10c27cc01aff9f1f90af11382b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"89bd7a623adb5354a5f8c65504e30ec84efdd85ed2a46a72c2401b213c912802","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8b94f2133755206c3dd7834edbf78fd4d7dc593a42786d5b82b69feeefc4db0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"8d905c196618d4530f22d1430069e41b0f742c52c337188e2d26d0f1afe42e3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"9168c97144a1d17cd7592e9a7ab07318188e5559433bd9a9095c6df99462ecc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"9de186bca5f6094a9f6224b75e41681f8fa5fbeb6c2779f1896ba6003e742fee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a016b5b38d2a4d75d53b6bac598237104a6247cc3099e70e2994bd0e3e4f3c67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a47196edb9e16f5059f4721a7f42f6ff2b4d49de1c50e00267532d5dcea13e25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"a6772d6c106905eb9f6a3066eec528ee5954761eb637001321b4f9f88fd5856b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ab318d0dddcdf34758c7bdb0bc2d5ce297d508c1f6dd190886b2c9185e339a90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"abd5b0bde20929e08f3cff92d0e75df98245e589a0b828e32c756457f0299f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ae1744745976b65d4485f1254ff1f7de0714d5bef862787f9789928eb0679c2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b0a448cb87c1db6abd793ac1e2397d859df5b317704c9daa67d5658b46c0d87a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b1da5cdf26e0eee3ab2b8e8be44d08620bdf20f8433bcd360f87f3dfcf2f9c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"b4a5a50af6009581dc90fafaa87f4ddba1bd3e00165b7e289232ffb8b1070898","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"c5e0c7d19a8e2736fe9718b51962cb6ca433e6e88b4677be6a3d840fdf36b0ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"c67f5535df3dc3502025fcc409699feb2bfc36c4adc644e0b97406a0edaab71d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"cac1cd023c3da045c4f4887f2d690e9268981477403935ba03e92dca7a280d1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"d09e517a2b65e275aa9848c4e08fda1f72ad388ed8491a5735bb912f6ee48eab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"d0d4aed9b08ccbcf282b2f7c430431f065a7afdf35b7962e9f9e39791686a77b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"e5745b7eb179164bba7685d4463315e3b097f4a638a982a2fd442bba90e4bc17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"e731596e866a4f1aa8f9bf3e441194ce1f7bc0658b366cc2e9ea533c0b061f8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"ecae1b0e2c90b770a76b856b5e8af7e32770584b190feb7e81bff82bfbf789f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f14e289d6a4ebfe3fb35f46d7570468b025aa7a11dd277e640517937dbc0a918","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f3a544a747d524e7c42a166bea6da176cfc9fd44cafd865f6ef95fd355051d3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"f8270308b8276ecf0eab70cb17d92106639dd7da4b191567abdb0119310f33b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"fb2833678dadd3afaa201320b7355bd70292569e1299718dce7b9367cb5b7866","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","signature_id":"fdc3dd15ff2799efe3a6bd04abb34c603047b1aaf5a9d25deb018df7ea20395c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"028e67617c89905a105cc4a612840e75f6897bb9addb98ab7bb40d8a55372dc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"134078396d0d2e90e6acf179f67c0c5c29619d70cb5b99448169bae9bf7a53c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"1455b7a170049f52e308cbcd54385fa67ff255e0e6641c47fd37f7755b51636c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"14a8c56985cacc9ae125bee7248507f8c5ec554db604d66afe031c0038c6c3de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"158f12004ca1915f2f3191b52a2e9cf87648c5b229999fc1a8b981b548f5d2ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"15b911e33e9c755011628b5fe192da1356eb0d4cb047227a8bc58def8652e545","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"1659d60c3f4e509c612345cd89191604a251a0a8078810794c41b4c4cdea090d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"1f815aa53ea4d4346365db14604b88f15bc2ed059f664fb14b4af3f3ca4e528c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"200a19ebdb17256d9331c2aa684172e1327200af8b213c83c2417edad907a31d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"308bccdf6612e595b3ab97a427e5b7cf09775b60ed9f62490094fa231150246e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"4d9b7f2662dcb7745409a8437cd36bb2b7fe3d3744f2311af178529ff0180473","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"5001ea95a48aa452365fad4a89be1585157725813537a20c155d45169398c028","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"559b8323654a159af3325d41df0b8f67a4560c8100c549c7010720e017f0392e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"56b282587877a0c992745a7e777f42c2329ad8e8d2870c9b16dee50a29ecc3fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"5eb660c4b9631ed92df772ec5d0f274a8e95adf3b7ce117f39890c1707545c26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"679c9425ef46695adbfae3247fef9b32a62432dc8b71340ca0632dfc151535c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"6fe2156bf8cfb75c6d8f9c2fee4e593a246239df1895f046cb9af18cfcfea086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"71fdaf900552131812fcae8b61a9676607004ea56053f4c3d2f760b74c226ad3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"74f3a750328c3cd542cbb06e73e38642e88e91149b817dbcf78144f7db362c36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"80ab27c2003ae673a88894b4f16c2b34be3d2886fa843507985c0ecf0615f388","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"810cbf6d7f7e2ac90d52b67a649ab8562d7f958a52dca5ad11a67a640ebe8cea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"87c4328596602d9fee11e1d5586b16d36dfdea3ecc244b5be95e0ead9e07fe5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"8f31a14086c93cf05fa58efd6f7ac6f6792c564a3a78418503408e590ce4458c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"9064132d524eca5a110d383b68965c649423f2c6a7a98c011daa34d30091d1fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"95cd434bfdd2f7a22f7b502fbf40a0223dcd791fe2fe4e9d8441d5a79f1e9858","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"9f14e27a49ec3c045e8184698464b340e779ebc18fed8b6d564892bad88dc4d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a075e6218bb7744e36b44936eb254dff2e82cba2a27dbf09d70bf39a3c405d3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a22cdef3c94a4f22d3948d36021a322a8dc677144620cb5969989b081354c899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a83dc4016d9d981f2ca25c792ccf0d882b5f2b518ba93befcd305baa0e72a752","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a847f67769c58d34c5b1c9f5edb54c406bd4ce3097b77e36d148660e89714943","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"a9516bb510596b90539cbfadae884a6fc2ee3e66cfb81c22faeb69b9fccd6e4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"af3f5db7f3b8aab06ccf785bc8dcf9812f403ee46c33dd1f94132db968f544ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"b06c1efb61f84c865ba6cce98104579878135b2fcb639431f48485564f5334b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"b6092b5834862dcdbff59994ecf5c15516918216b25624a1a8e34e8cf20ab91f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"b6c4f658fa4d8951a503f16688a96dec150a63ee3bae940202661178db6f28df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"b9b567307ed244c17f853e374b1c78b752df4906beb3ff703c0d2473f1b6fbcd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"c05362ff9e4d0172e5ae29c0bfaffb620c3a19b99e013eacfd7f50ea0a619126","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"c6f8e5571aa54819347dcd74c64010291cb3bbe12b83892aeb0d039233f5110e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"caefb865d0b3a4db06386013abdc426cf7ade03892faf311d759877af93b2b20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"d507d7cf0f593f52cca6bf6f56c14130a86590f9713918391811fa4a6919b30c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"d7399f90009945d8db59430f56c61cae27445036521760c591629a465c830d13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"df55f6bed89a1af8c0cc697d8652b30a0d8b6414f6f83f78d0039e476d8851be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"e0ab066f324eff7b3bcbcc6c36d44358b76341a4409a3592b7bd7df88172d18d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"e1c82124e155c1115d2dbcf9a50d4bc2c94263abc4b30689eb04b100350c6c30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"e4f0530149d9c515a8bb2e10bea920e06291e06a625c67e64d57fa1abd5a5b74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"e4f7cef6331c46e4fab04aabb85570a54c299a0ee51f44135e8f8b3cad753c3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"ee4aafc530fdc6d6b01c26778a1358106dabb25097f0564cca2e72c97f9d0658","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"f165c16c6c0a25cc2f95a20d410a0506031f94afecbdcbe9fcb801885d325f60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"f39ebe4a0a487af20c13da8463b9fdeddfc15918af2a9bf235cac92bc6528e5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"f644f319f19d576779a8299466637a992ed29ad34884b8a2b8b76112b6143919","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"fd166ea65a29cb24c93f31900728e61a500d3e97d9d71b119d88d8174ed940da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102533571657863168","occurred_at":"2026-09-22T23:00:41Z","signature_id":"fef37f6e3d1c779f49824962c264c15d6c4f20dc311cabef868b4444b33225d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"03b77825464cd5d0f0c913cdaae7947a72c014fdb9444903e3712d0c0fe522ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"06d9be7494f12960fdbcec7b743d661d01c049e05bb9bba0d808e823945968f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"0ab4166b7eaa370dfdb49ad732a32dad0ac91724444ddf819aa3c29f3a9f6a70","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"0cfac70bd1340931647bfba310152659cb81acd7ef73020d9fec20384f21a480","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"11dd84afe0645df3b655944fb22194c0f2e58c9268e44c457850cea2b7c34e78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"1438e991fc10aac23e7722574797bd29edc9098daf2172e0fd8c3ae2085d262c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"15919028e1ac14fcd30d83d00063e34c3905a3b42a1fe2c678678bef887c5398","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"1662695320a9d015fc05041d761e86397e300ef2d8aa4efd29dbe9589d289ba0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"16927c15887b574382c340bf94b042f4c251206eb455c46dbf767b8bacdd00e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"216725343df0529e9b36f0e7e1e13131bf0ce5434bc431bcba8f418e8e64c19a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"21ab44da72b7e9ae397267d0f0a8a18bded933ed8452908aebd7aabb096785c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"2b9093bf5c45c0ab4c53608afd8efb00f9e181eec19f65cc03ce4c6f28da0828","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"2d70c4931be129673294f92930d75ded19384939b39f6fcc68c306fe7a9a8917","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"31f4483198750e6aaf7406ab17a19a3d028274697f1c29c9b1c89a59d30f4bed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"324c00bc6b17f4c27a79bfb45836691b50ecbc4134a78456ae7c1b0d269b1e69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"35acdd5271b6215a49c6cf08b71aced0f1d2578371ad1ceae5b07fceff8cccef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"35bdbace1efff0b1b9097d9b772f2e08e1c597b678932a37cc1b7e87d68c5529","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"4fa3bb0bbd8451703ed73fb305781833110a4c051383b3123925afe42add42fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"5fb1c2770708686f8963accd704a5ec1da9ef9cc1f1cd732cce61f719fb651a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"67553b55a2af77864f0735e878cf7f4a32d0f0ca673ead8a06f3d92d8594e2da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"6ba691bb4d98d95c22f5be1e9f862a0628528e2a58e45d0cc27810b53fc1964f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"71525a5147dd46a524bfebfb7a7e9ac475ab1cd78c5c49603707efaaac058811","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"795c08d8819e681ed7485b9c8f3dd820bc16a481608400dbde3cb44146d78088","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"7ded978aba44423367e409626a9a3315d57d345e256488391ce34265653bbcd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"8046cfb505c99b60ca80e943e2bf1314cc6c403f4aa343db74d259ea4a6e93f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"8dfb46dbacd72001834d323a20b869a0f996359144e2afd458b2448b4dc98cf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"9094d9c9d14a7b83f4e3a161a729b7277c10c8d146f7017e283f7a8344d42a64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"92f59da349e0622f853a8c3a1bec0a0e8c4348115990bbdcc3187438162d5e69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"99c18bdee1e1f06cea990db26916a8c65ef3653ce4911d8f85a520ab39ee6cdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"9c08634571d13f7a120ae29ee62838a0103a629a513f3f84e21651285d4b288f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"9cf81b6880288565cab800db518d46590a69e97af1a7c21f4d2571d4e91ad36c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"9e5b3990ed8a6ed539792fe22e7870b8cbfe2fc062043db588d0f769a6c0f78e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"a31a984f939916a0796de23b382a377251226044e65fbd176755803e62ab7118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"a872f17e452fb889fc8ba0a1d6f796a5739c45d0dbddeda106b88f5f9d119623","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"afe89bb70726178b639cae42abd86fb26a45875403133fee89c17bb3f0dabdc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"b118fb558f5f55539b11bc8c16f10ac542da724b4d40c700527fdb51e92324d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"b481b2d65ab6004bb5d2705a6a89dbd0e37b8346d88ed8d9125cb85ab5882086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"b4e7af59ddeae0ea6bc250fac9809dc26653c15bcc599bcdf62dffbe67f2017a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"baeffd426c735122a05b9160488c4dbd60bc4728a5b3d988d4e82aba7d0786a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"bfefd8b73a95855d933ee982b50b0f0e5e07d0da48727c260ed1c8f8648facfd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"c00047326594e77e168a68c8731720f644c8614a7bc5587fe8aac8ff5098493d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"c09fb79396c6290ed844ec65b79dc38591d982d0e590dc59031e5715e882b2cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"cd1c7583f2647a6092127bb87aa94ca9928e0cb7d3b98f709bb1673f365c5faf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"cf9272d14bae7f20121b2149ee98cf928951ebf46d29b57c025421aa4ae43d1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"cffe34c5297814524ceaaab2392bfcacbf1ba1c268e4ea9471d44f96f9f151cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"d3ca56e7987ce4db58ef5dd7aadd3fd270620b1749223ce2e302a818b775a91a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"d6230e5e7e710e0d42bff7a6002f4e38a4b5e9da3efde2e66bce2f28fcaa7db9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"e97a9ed671fb6a24f0766a2b032cfb0e6df40f68c200b713382036952f0bfc30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"ec3a67d29e9d134e429f91cd72b99e2438ffd5f649f95915d71f08fe6389f0a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"ec99d240f7e03e54f55c1ce1e3f244fb13a98e2e3e13f3340e7c22ecadfa7ec9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"eea94e83586857a6e7bd796c43608d6810be52d625ea8b1c7bc0e154a36458eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"ef22972e47abc2792176e133a01c85a9b2384bd770dce3cada367ce224007462","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"fa31b30e1f0f06a0d18a6453b7dd232180a117910392ffe556124807a57f5051","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"fd2a8703e1cfa9007dc900c3faf403c43e94c2f939e63eb0a30b4489f161a1ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"ff22fcf5e35af3fbcc3cebd0bd3dd0c5eb072b99885e355b90e687d8b5d3d950","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":194,"runs_affected":4,"failed_runs":4,"run_impact_percent":12.121212121212121,"anchor_occurrences":194,"after_last_push_count":194,"trend_counts":[0,0,24,170,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-24T05:07:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106f760\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106f760\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13100\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13100\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"00a20048fc932b0229905148fcba92613e4cec7d16f2f63fce5ced5f3389d702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"029ca1a09e038a2fa4b3875ee6b1d8faaee67b0d1c74d5a1f35e3c471583ada0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"032d1d7a3e0d95f0228ee1c11b1d6af914ebb5716a5fa59eb161c8b7fed27753","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"08547dd081b44323301158c280d0f3434cd05b357d34cc1bffd49f97c29ef513","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"0ef524766d9390776ad48ecbf26703ecb351c6ee97b9799357d914d374e4f02f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"11b0e5a286f5fd7f972ed8af20e7424d0a69e619e29ef40e658a0fe7ce3e2fc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"121a5f9e10fa93bb1cb0ceb5ae519962392d2b7e9720f3e5ec5a5c3456f46de3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"1264aa04996713c2ad0664fc180fefec085befb80e6a3d7d8c615745c0cafacb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"1275667544a960574ea11de5b15d9cc89597bc9e741df1ceb42178fc733c8b11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"128bfc4ebf3e4f4ac7c72ef37dce1d6f1d2972dc28ac3084ac0eea3f1a935364","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"18b554797c4bf7fcf142d9ba43f2d0e3c7d0231faa68c4e122ac2697af20af1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"2ea82ff5e0105ddc39162bc3a1fc69443f8429775e71a60d26178a993e537cf9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"3202f84c16b137e078b9c7ecb07b12a42284442eaadf1a57d9b80732965d76d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"33d5d93bef7910045cddabd91ed8d0f16ffb3abd049de20101412caa4dbdbefa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"469958b60afe99ebd686b471f988b1e0ca0f744e31bbcdce7e52c5bf5c7f1549","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"4834d7a7ce86611f500d1d72078b83248491ffcd1d930005f197e68c04a38653","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"515b93d5e37ef059a66fe272396b5e973c12ddc43056fe961348a3e80afb8c97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"538131ec214de2b12316575147e1e84a6f41ede3f84a3e93afdd963b3cfcafcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"54f0aecb47d52922007a03672ff611741b8fc5034419c1f438b22e2214da6935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"586af51ee2e7cbcdde4edd35a4a1a29ef19ee26309fb4a112ed256320c18c4de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"5c5c223fc846287b6e5184b8ab2b2f4e1d090891bd7357cecbc12498cfaf9579","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"61ee045800b644443347ae3b2331e9a65e1d479e962d730dcbc98c53d281f3f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"66b6a8c8ed850f996ab5708c37e3c8119f622e5f7661cee41c168026b7225a35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"6c9e3e00c4943e8c6ae844a720ee3bd59a661f127a0c9f3e1eb2ce1bbec51ae6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"704eb4f5538f36e002f0ab88d0c2ebd5288bd5bee60926b525641b1c59219a62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"70890c7ae52a1db455deec8b3eb7490070c782e63310b6e7b685353def12711a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"74310ad621787aad91f122d4ddf1a6955a3bd453e42f4f9419b9b40e9ac8483b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"759c7aff752bc2da8c0bbcbb05976cff2e0320d40d2167426639c5b03ea648f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"7d08c2187cf3f7a85081d3d94aa55ae175cd4f81026e76db422229194afedcc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"80d153140ed2a6cc68ab892fa4a4c45411338bbd76b2c74ca7b9855bd7a64595","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"88bf0468667696d126054af4a76866d6e508bf841aaa491e564e9b9dc090b9d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"8a7dd97750d4afe253dcc7f912d067e5dc02351e57c7ebdd90777aedf3f19660","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"9b1e9bfdfcae58a6d83f60788f5cda0dd05af07f85b2fd0f136917dfeee0ab3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"9b2aa2b4206e53932073fbfb64d37b9c7758a12a200a2900f869b67ac71a624d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"a6bcd28d73459594e929894c498d1e701b6c725c8fc5e8572366838164afdef9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"b3cd8ed1378cd179b5adc6dba47cf4e17d2d259a48359faed9ab53bfebd698ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"b6d7774589a62ddba3343d513d47008b24ca7189da168c615a3350b70c6cea72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"bcb1f165d9dbb4bacc1f9a57572c8fb656f3e7dfb80956cca46af347257d19e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"beadaf19cf2680a9bc17ebe12a30ff4c469faa490d5bcc9d5ea6546c43f5c8a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c07a630ddfa4bf5e5c0aa0512829f6419194f24b58766d4a80c2eb68ccd03946","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c5ed5a71619d53dc86b567736741d4b55c60b49cc02e82c98980eff5d0786438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c680824be3d2257cbbfa3f9069712276e088e630af553b971685d6c2d068c191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c83af701810277995349dcb52023292d5025daa4c73897a842f01114b802c577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c9904e37a7c79299aecd2c19efee7467df44886a34a6426023ae1408ffae1aa7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ca6a37a8d8bd66ca90ec1156ff2a36ff215e8cc3cee04729bc0351a1eaa5db8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"cb62db27ca09cf506870028f97396da42cf6ed7737e04d83b76d35e31b696eee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ce3b2648ff9b027b217801bdf840547004f2d0850e7343af604b6b1c4b1c31d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"d95ee625f921273f4468efed93cdbe7bf7598dbc0e5fd1356a3bf91649094680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"edbb12cee0cdc493ca116717026c262450dbb9303cb1deb1d5bae75c5bd6380f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ee95ed63c942a4fe05405208f698b779fd419a59453da921efa0d030f65c38da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ef6b465c319e2089bf73fd85a6ea8152d3ab980148dac6a645125758ec975f33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f47fe5731873797ce03d1d26c02b76cb7c541a6c274c295b42b92e5642fd89a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f5544d328bdba7722d736b6d0f52967b438c0c26d6553943b71ca1a3207e4a31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f78aa7804dffd9c91eb7a06ca983c1ab58ba6e852310a8faab7a08106948a7f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f929f4bbedfac90e88e7ca84f91f3733417298186cf3e83f4d4be17bfe2e51d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fb5ce0354606977e7e3791538a4b663a81b113db149cb566343e57faaf207eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fd8dce044fe3c3ec2ed6837b879a1f543587662baefa69190a60f6b8acd23377","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0304d759fd7f723fc6fce46557757270276c6ef816162705b4c141fd03f33c2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"037c80fef56df03fab8fd20ad0d7c86fd678553fe25a7b65681038b32e94955c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"07068c99aba746939ca0e7fd85f8d0268d89c19b415bcc1ee9c1c9a86fbe9b05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"091b2e0d12e6d8ebf4284168ae4c1d8f0085c61f2ad326ef5833e98ccf402f45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0b5092b1753b51d12ff4c29870adbc656c13e6976d861905264ee7910638f41f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0bd173944720937f7045f39d5686ebbee16cf55a1935292dc7a1905cc174def2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0be8a0f9e257eed68f104074868b8b17f7d7098c3a69d645d20758aa0235f203","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"12dca558bd0b511fc4a5df715c750446668edb1880da33c19093890f8b547639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"1623bf2f430076fdc11544face0d86cf5da4f75f4a70fb76b1fee7e3b47f839f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"23b8a93a781e18d248ec90141ae1a827a89330a7e7a6023ebb81f712b1aa55d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"26253298960cf5b479585bcc88d9b58ed6d2ef483db264c7c7260d823ea27662","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"2d4c586a289252410c5432cc2b9b0821bd3ea82888527dba0a2f593c88ba237a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"390eb6735fe63cb426b4f5ffec27b7b5786fa5222502d31c5b03c65ff6af65f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"3c862361541ceae5c062b4ba0dc064b68cae0b5d598e706f6762f085e0607306","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"3d478be5b9189d62ed2c66e9b103f81f7b86937b57c08e9d67e6ca3c30555f01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4d8344447dfa277aaac9f86488fc2fba58be5beec2d8d53a92b1f78394fdde32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4e29cfc3d96fb63ef07b9b320d9429e79b35c452ece26e453a45510107fb8132","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"502c008e6a6776d9a96bd10fd01cc6e2363833b568e55b82bb09d9627e16887e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"50fa57f55c832fa17b6f7248a05fabef16d409042f554f1463f8a8292757b10c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5b26974e1da059929c65b663e256e8e0b0cadfd7ed25c9ebdf8345a70426d39a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5bd96944e5c33f0e4a58b08b961f8094f9e02ddbb7628a6b1e92dab670f4873d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5ca763c04ba811fe44fa262d90c103eac78fc982660138b2fee9f970c2edb86f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5ef6cd0378df1b88f9a1e441ce1513797d9143870e2e548690d877816da13ed4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"635d55a9178bb2083d65e2a4ad10093076f6eec948e15b30f0eb5b0f72168cb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"63d34142a504e05f9036dd1c69be15d31757de56dafb77a0d0f0177274375c52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"641eb4af1997048f835df858f7f7bef584d41aee6bb04fe645ddcf1e44aa00e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"6722686da3723d8b0bf2c0c56e808e2f4cf683cdb005fcd9744c5aaecd3e702c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"712f9dd3b637bec2b5625a0186e7395b21baa4927eda84c10d89e02f5b9396d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"725f05032bea5bd0ce516b91dba1d1346b457790f2cf4f5d22ff28e0e5fca8e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"7a55e2a07f0fa2912503eae7151cf0970e3f6b2d6668fab56021d109adc9d5ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"7d875d3f62b6b594e123ead8b904ca4dd939eeb32cbb39316567bbccc9207e3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"83f41dede741122effea9bed86f31cb70cc79e3fe60190c74a81ccf095575a1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"843855ae1f0f4f6cd0e66370447e4b9beddc6fb7e68d7e5e02492021371b9652","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"890e1a56439cee473baa172f8e4a9c46c0855344ecca13d7f575a0200da7b939","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"91478fa5c60e4c1a9e847cc7a2a672128b7fd9e86bf7c5aea99592fde71e73bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"925e74b7b1612e5eeea6fcbb8dbc566263b823de593165702c918d865fb5320d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"96b896ebd4bd97e76579fe89b0bf3e2bf9f53a8625a52c8127ac1a1e8e6b3b60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"982fb021ef82674968423b11c2470423c0daeb157256c664d1becb93e8210a75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"994e65619d43c6d0e316a7b351700514908589ec59750bc0f6395e6be91c7836","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"9bb8908c4059fc50132ad3029a7c2b0cbf42b2c3896b45ab04df63c56500a523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a0382390c3701f1ad5bf1bd591fd3915695d3a07b9d0cc99388388748d139457","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a1e87d9c7e66aaed64c7f8954d9753bc0574003ab4b1683459ff823c1d03d7c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a70700610a1b0fb554b19357597279649c0cb3ed6f5aee8c998069ad0ae49628","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a7ea24781db537110c35917d98b4f72ecdedac8d30c9c33371781202312a2f05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"aaca3b66cd93923710b64ada020932cb93aacc5dd79493313bc4c64ce07c0aad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"b3fbbe913167b23f7cddbcc5beb175723e00dfbc86894399f39245e5c30ccad8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"b890ac9bd99e079a6b5aca1038203032bcdb627c5eafb16e539f64e5b257093a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"c17a5e36a9b108e239e9cbfe37ad374080446eb38001b2f19c480dc30ff0f28d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"cab3a7cd895f69adf001c052761d92c86ca6c64727e980c568920541d66b6693","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"d6768673dc1158caaa55d77db95d18d312042e5c93f5efa86e3fda846977c9a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"d762de026942b3816437197f33b8846d8d2d9b251a4615d40901dd13cdf8719a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"dce118f0d6bf88fb6228496d672d2f34dcd0097ae26939c9a39426a75b29b854","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e4854e50e27a97b30d706a8853005f6b6d013f9d8ed46f6cafc31a004843dc2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e720cadf1c011d298dc69fa164a357fd8c779a17a439688ca3a04d3526fe32d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e7f4fee1c466f0459af35e40872aa4b567ea48855febfe78971983ee7706ff97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"ea1a9664879960e3782bac1bab4881ef33ed58c68b27f3b3b3ea9ee942bc8f8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"fc6f49312f9cd176b6a5c0c05d229dbd5b8a414551493a3fe3a7e14f32ebb667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"004b42a6843d9c599be4103c0b3a7bbc53c9a341d8d2c16ff587dddcdca93fea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"02474ed7bffe3f2f3efb832206e4ee8bb2657d3eb50cde44d64f37e4e025c8d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0a134035c835ada31f01131908200d3c87d62d1fac56b860bc5699a1bf6f2546","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0b7b3b363d12dcde6dbe7dcaf8d1bfff05c68c4872ac68b624b4b5dd24798a34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"21af537e81ae2811dae2782f1541dea999e6b6e005eb5ba0438334b0110de505","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"24a84c9669e3bf30f2efd1876022065618eeea5fd4fc89076c6ac044a71d108a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"292ecbb651b87ea834cc7055016245c2d8aa052ab0fd931987019f543f6e4d6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"29363b364d248244b8cab05224e499f828ca4b3558f6b271121ac89221e31f80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"29738fe680afcd7bb868f6b4e7bf0e48f78b9de7c821820e7ce46f1b77273631","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"303e71f56a7321ff7781f70fdf21e43e15eafba5a112fb795a421560bf8687d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"378c2c4f5f003f5f14a5587c4fc6d847ccbc184c6e7fcf305a9df629726e8dce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"493df6fcec5673d5971d28b2495e7b60a93b79fc07aeb785fb4063e1017f859d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"49eac6d4a36a30efa10c1b8b1b8d2d65876ce6ea80106d791d42cf6c03bbbe8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"4b1f37cc80923b8d11e34699631a1958ecd1c54c87c846691565df3c140d290b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"5275008bf6e2cd955a92ff0c773682149b95d29566406f66e1fed2199df22e64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"58e4dfcaa596cd88fda970ca6914db4000b0185d3538f68ac836493847042863","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"590a7ce9c34dea2488d7bcaa818aa04f1e7333c2c32aa265eb01f529e4c09168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"5e81e681dd240666f7f73c328e1314295bc989aa39e4424d393b9191c79a5847","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"63857a171b9f09d8d921021f794a90f8ab826e76cf935d7a69f3776cf78381b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"63ff74a61a4695146a80e07d486ea919d166ac0907ae451cd218aaed86a6db14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6664222cdd56fe98e8670b4874f17c78c7d2168808f807394b24f384709545e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6d4c6f30d7a0118a761afb53daa3e256cfba084c934cfa3569d0253ab1d5a261","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6efac1d441daa5ff54b4d89012d20f08600df5d158a441ec102e24943fc428e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8168feff532830edf3ac3497f6ce1e0b657479305b2853c1a48cbb283d165e76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8254913197285306ce0204858e98e4a4547494803895048d378c8049c527c44a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"84c63d590be5b6794793c8c510fecef7965137dfae7568e99ea9e7ba5cffbfc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"93efcc50d250cefb160b3d3164d61008ab5df04f873fdfcbbf699d728ee56461","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"9d3778e9e95acd335ee5ab579b70317fa3b0f3a32b64bfc784b8aab8910d2702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a4a7b00e0cf6167aca09a6c39407324ab2cbdf83285b14e600ca3cbebdae2762","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a6fdde63eb30cf8fde25cd7079bb5bd9247a18fcc4fc428d6930daff70e0c43f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a9db7c4a29702a102e7f5ecd1187fb046e212b54e149b8fd5e9d352e54ffbcd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ac85942acdcf9dd62304e20814465a73dab2c6546d5dfd1ec23a0709cce01819","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b0ee5240012c5afec47ab2c288045073ea38119a5b50f900847fb95919697b1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b27d8e1b8d937857f4f4398e505b9f826ce2b0dbdcdb3fdae66e91447431a872","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b49424c3e5a898d2542be3de8a944c781a1c54008483164cde2612601cc567b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b60748e01d3770081b9092311833c353d24a585d648763d5987389842fb13c7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b7d1a1e20d148974b57db9b0bf46b2d32a4fe75a72b18053adfcd3ffc10008fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b9b552cfce8aa134690435adce391de22a7d10909a7f0b0b344d1de5d5f13741","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"bfb111b44b68e6d49553ea155dabf55537d9068f181b21a0605c2d49bbf1a5f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c3a66ee84a59916e6ac4929f00304fe7deb56ae962c7df382745b1e42339ff55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c3c4a6ccbac232171b2b65ce786cdfef92bbef99d093fd8e6cdd95fc35731f03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c80bf9057eb592b0d78f55fa548e2f6ce83aaf5494a5e05657657b2e3f4c11d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ca4be0c2537a5986ad337e9dd7b0c174dbbfd555b40b995690b3ea0bae599488","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"cc5beb5e38150b98e6a39e86bd07ab29baf902b0225bf3a00a1113d79a08d467","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d32366df33e5f54aeb4a5cac0b58617ef9795c05332cf16b4e3b9f7d2f335ba2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d71fff82c2dfe656568e32c9670f076351175cfc25591cc171d7af2c71095862","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d949d00c9bd448d0c2054b0f46eb2a81e5f314ec80d1542febb5b972b95b51fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"dc85a4e926dccafcd7a84d59a6cd891fae237d24dda05b7aef3fa9e97dec9da2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"de91715b5519c3e9c48e9f9db31ee57baa410e66aff582268d9803787a051a40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"df7c02235ae5a45c83397b743a331f854e72bcbbb98b42830177e7f9d038928e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"eb699857478bc226c6dc8aaf326a1110ec3eb808736fd153bc787c847ef8fd08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ebaee311f476e862bdc8a9ed9fb42966e18b1240f73ae004a1a47556a07ef340","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ed72ed9221365984a17f27e8ed0e35e5508168087855a2c0749774e962bc35bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f473bdd0d73590343184ba03445fe9c69d83d507ba5dec3b6434fd8b6dd23c83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f4c7d50db1341281f9bbc735079e7958656096dd31c71f4c0842b77c630f5f87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f939e6af64ed35d589b26c26d6589830f2545bdd650fd5e4cd474c0aca8fe905","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"00d3d6c70857d58cc7a77ad3cbf60f9749e64131ae86e9c5bbed23a03414b07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"04f03534a2c11433842c187808432c4fe56b5e127cb74b63d712bb47709a5195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"147bc66a9ffa2ef0eb84aeffac927813f9dbc0de655e0de415f62f1152f420b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"1658b955eb3643def75168d8be5b7cdd2bbbdffba30215221b837273eef86550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"207076026b7204986c55fc7402443d46f35a2e59e1d8abc866ccf5d15030f36a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"2577d97b4cfb09d0e29fa89cfbebf02bbe5521e4fedc6b8305c75d3a21b78bd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"29b4f2a9dcd277b5f12403d713f2be8b0efe8f05e42b48c6fc64c2d68594b63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5743f1054d0ef1e157783ab637285ae718a794a73ec3f9d8e1cd66e8dc185998","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"578200e0d6235a987906427ba8a60a1f9bad1525471deb0bda8be5707a26c15a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5e161d805837a94ed6359e3e20abe23d9621b72c256eeb4160963d2f2ded8efa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"5ebbffe49b3cac6c858179fca1ea96cf96ef5a9cb1c80ecbbe6a096d6439ff4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"711a23fdc6554e3f11a83326688aeff48062de542e6bf7ed6bf7c7bb3c91aa4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"72b7cdf642ce3d305cfdf2201f3999ecc6881553d5e2167443ac80f1a5e34e81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"8a3707252faaf14736b2be9fdc31fbfb7ee6fd9a2c628081309edfdac8fa18a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a243861fdf0dc90f346c36654a0f53ec596c9df69556eca343abdfd26653e306","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"a96addc801edde278066d55645bd6687bd80418d4c10c554605fa06785f260d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"ad868390107afef487eb4050fa0c3e0b12550c6ce4d10603b345f5c954ed7fb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"ad8d9c518342d140737ad5d4a5fabe301705900959b3e86d1f3308ce70e5f962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"b5d93b8bde9091dbdbd0277f382eef3ce49ab826cf8e62f6b226b70f017f85a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"c2e9c1eb22fec11e3787804927eeed16352b79cdf1428605ce93e959bffaae09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d2c063f2267dd2c0079f8d444aad43b12dcd51b7cb090dcd5fcfa03d5d526369","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"d4b1caf980e450681931897e537f4227ce5d0899055d23d5f616842e4e73d6eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"e0d27c79c5899c123b80405656642920e76f42cb9309a89bffa6098039e41580","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","signature_id":"fc7baf411234e091570207b597ac8f8d793f9adeefa4cf31f1037c197fa10337","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":58,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":58,"after_last_push_count":58,"trend_counts":[0,0,0,58,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T20:07:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n    failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n    failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"01df788dfb171c0668bb7074e3f88aca94749f84bfc4fdbd1a7fd91e08a7cfa0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"04637e18bfcb3f9e724928ed6c4c1cf581f36315f1bfeccdf3f80802e6356d9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"08024cf849a325cbc5a336512e48936583c3f85cb4a3a2c148f103673d901cfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"09ed45becaf29978462b1c3b2ec63dbc0323dfba597e299f51b743ca27d93962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"0d104c19c86ac40a63eda038290eacd0119503471444454de0cfa1dd5d007ef7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"12e68ce9f4a1569a39aba64a40ff879ce989a6ff69156a6105233cf4609da0d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"15f5f6ae828a46047c858aa69ce37554fa6dd71efe1a2cb7544916905c1ac2ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"15fba99f85a72c2b953f1221bc696b9bf82753423dac0791fefd94f24e82a3db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"1d9102a2f82649aa9433fd7ca07f7f8568768752d9d21e09bf205c33862544e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"1efd0a59df6c4ca49a656f7bba9764b2b191d6d3446eb06a4a269c70541d699f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"21ec5f782bbbcebf9c1bb08d8f3860277c036fb88121f67bd7df545279ec079f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"27e7bc710c289477425f6c73f134a06ae089c56c9d0c553da97d5810dc3c16dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"27faee0e8e7daace9345589564265c839002d698d5b6bbaadbbd0fa8335b0ea3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"30e39a8546827489b79b43d722c71489114e6cee327dd845c522ff10e2a9e7f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"3111af3121a7744fec69a5d201dca82460664234a0796e6e4967a0c4f1b2fb94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"3557fb0d5e8fd6771781276cd1bd6b8e55cef0ac95abd5a7179bcf77ad589c58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"39d2e978a6cb835985b2a92f5546d92872a269f4e9257aaf79741af5c03d26c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"53ef031d8e131f3373054e208b7c3fe7af1f963c377b7a456e7885180b60a0d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"5603a2702aef8190ce731f49e3f6da6f4c67aef7c8a2f54bf3011d9751ff4929","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"5cedb6ab199d625ac70604831620fff130147fe82ac076113745bb3622573301","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"623139f8f571d2f5ae27e6d75e9b76c6e49ad61954e8cbea14bd347a34071a46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"708386bc40e3cb953a5fa1b68320e189ab8f616758c7e5e8216987b789b6a323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"736869058c088afa7ee0e36a8c8df936c6ebd0d3051c6c1de9701a4b9d0d1faa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"7eec216fe53eb861071172ca28122a7fdf1495ace014b5cf90792e298e12b594","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"8d6a29106f6d7f3260c9698ad13eebf0c73b422c9bdeb8162463a6c655d9d521","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"8eab1b097f117f5fa387ae53997393440b33b00e85edf8ebaaec122145cf76b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"90bf1a4bb0f55805455582cc5075cd286abac466f6ba8106c516801ad7b9f73f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"91e6b9daa46d3913f91a3381fa2d7d1c8603487001fbb4805d640eea2c9816bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"97c7d0cc26d3fed4b372c2504b2ba57618874ad5b3a6f3eeea32900b0bdf71cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"996575218bcbe29d31c4ada2dd5e23243f75e74c9955fbef49e8df81a1d3822d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"9baa846c847b6340711d575cf15d5a87dff938dfbaeea48c84709a07d2be7a00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"9bb7d0520b881857aa31248f17345ac080812b7c304ff8cc09d37fcd8e3025d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"a322a676fef23495954c29cefc197a2d50b5612cdc27cedbae8d3022650db26a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"a83f70703c99a3e6b99485097810f7a5ce10697c954f41db95a0c6d9c746e8f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"aac0a4fadaa81917b45017334cfcee4d41249c967b1a67dc637c0244f20f2dce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"aaef8153c4cd154e06971faa50b02660bd9e7ea2bbcc29a2f34394cf6867b48c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"ad047b313cd46ff403890ab91105d7e24c9d73411af2aa2c234da58252bd96f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"af5f6dc1f9242795b07b341d7cdab3660235226d8b0ef15223cbd5e68cdb26c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b24288af3ae287714be949e08bf57141464dd56c19ed7d6236a3c80a3dac299a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b5ec5970e9c7614d527f5257f02b1a83f847618cea746349aa1317c1475e49f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b97260f6b219bb2a5cecd3306c8357f6c318ddfbf92f98f909285fe3dad5b88b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"bf64f485c1cc87388187d9c3b48f24e4866df71b94091b1f848430665c4c6977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"c1f6def3d468e1d53dc22a82731f7fdf79b4f1e68b13db8097950ff0725b0301","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"c1fb319f0d065f046fccee3d75086273838a8ab728564590b0d0354f67fbb476","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"cc0de1274b9ceffba6b657f32cb1765d623a4c8525822e22472a4854ffdbdfd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d030cec66b2abe74848996b1135963ec2f1a5deaa837b7c5222ce37dd6decfbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d52eea2a457782a8ea36566d49d84e7651cc963ab39843912377b9d8c44a04c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d8bdf92b2e9204714596bc6a2a6ce29e52d30033a7f33ed1bebc44c552ce7396","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"dfc4c8893a466ebc947358160114e3f5e45b2ef1a49b11beb24e101d649532cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e10cca4cfd9139f298f73a8d53b94d84b35a5fdc8d47d507968b47ecc585cc9c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e2c9151c9266bd39737c0145a83b25202e796e3680f387fe62de4718df836011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e4596221e8bb2095fe0e124bbbec22ca6138f5070eaf0a73880264e0ed679646","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e56a54c25506fef4fd4c8605a192ca2458a0dba96b5fecb9ab73009654f3cea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e62df2a6c20e4789165daee30b92b9ff0d162ba6cc3b030a9068c6492677d781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e6f223f3f936617d0a8b98f8caf439c033dc69e26b23715f31b40c276bc5de9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"eb955423d40836c1dcb045a85bcda0cb03498e55747b2de9e2048325db028477","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"f4de3b4a6d6871ebd4e70e0b4bd0fa3064ac875b32d2edd0ec307ea339df236b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"fd1596bb61848aaa7a93806f7b61359b36fdd12bacf8c47ddff014580b73c416","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":48,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.090909090909092,"anchor_occurrences":48,"after_last_push_count":48,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,6,41,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdf28j9mzrjf/cluster-nightly-5-1-ds89w5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3e960\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-ds89w5: failed waiting for cluster=\"cluster-nightly-5-1-ds89w5\" in resourcegroup=\"rg-nightly-5-1-vdf28j9mzrjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\",\n      \"name\": \"4cc98024-5165-40da-b846-ec5ee553964b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:13:16.920404751Z\",\n      \"endTime\": \"2026-09-29T00:42:21.556617536Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdf28j9mzrjf/cluster-nightly-5-1-ds89w5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3e960\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-ds89w5: failed waiting for cluster=\"cluster-nightly-5-1-ds89w5\" in resourcegroup=\"rg-nightly-5-1-vdf28j9mzrjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\",\n      \"name\": \"4cc98024-5165-40da-b846-ec5ee553964b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:13:16.920404751Z\",\n      \"endTime\": \"2026-09-29T00:42:21.556617536Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kq44rx at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a482e0\u003e: \n    failed to create HCP cluster np-skip-minor-kq44rx: failed waiting for cluster=\"np-skip-minor-kq44rx\" in resourcegroup=\"rg-np-skip-minor-kq44rx-wrs7cdgqmf9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"name\": \"123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:09.384523535Z\",\n      \"endTime\": \"2026-09-28T00:28:17.690643945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kq44rx at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a482e0\u003e: \n    failed to create HCP cluster np-skip-minor-kq44rx: failed waiting for cluster=\"np-skip-minor-kq44rx\" in resourcegroup=\"rg-np-skip-minor-kq44rx-wrs7cdgqmf9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"name\": \"123e618f-af28-40f6-94b5-7da4afca4a48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:59:09.384523535Z\",\n      \"endTime\": \"2026-09-28T00:28:17.690643945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02800\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ss8z8mq428kv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"name\": \"5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.434225001Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489114255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02800\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-ss8z8mq428kv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"name\": \"5b586f4c-dc93-44d1-b18c-28c11d67faf1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T23:02:50.434225001Z\",\n      \"endTime\": \"2026-09-27T23:31:59.489114255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"a7eff7ef2cb5084bb89a68bdcf135801052c5704577a8158bf99956be0c5976d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"062e7d036f7e68f5186271d94c1e3a10b9458a3d58b06cd0e478defd07b47e4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"0d62eb4a49f95b17a29f90933dfbd4d485316c175eb1797cd4f2382596e3a05e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"11247ce48003ac7a491f84d736813861aaf0b3430f2681f5c7d7b51f7e3584ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"119df034b49750d6f9010e8633d30b4bbb66194fdc1ae0c67ea3b39be3b9c543","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"1813a0655895bafdd0fa7795ce7106f0e053c012c9ed0db0b61b9a3da768370c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"1b3635689317c3e03438e1d8760fb8fbd558bfaf8f0969472e313ce9b7e51d33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2149a88e6259e337c7dba379c74c61630a619c61b816100de91023f80eaf5155","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"23dbabd9a7a25dc4e3d2c8e0a518351cb3699dffe4de727be870564d6f2f3e27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"26273269368dd302144ff0e225c732187d4fb4a38f6aeb0cfd6837ba8c834934","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2ac5f8dbb6cec6887c383dfebee159a130fca9941d88962c9ad1e6a8cd766041","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2b107b242908c25a7edb0c1f9ed1b123d45886939e1691d3ccbe4936336d4c1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2fe44cc5813d3d43cc5b78bad763293044af3210e10b6ee32a7f52568d9a44e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"324df14529a82b1427108ceca83d1a217a2b34761f00dac0a98207aaed8e2f6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"38536e6a87358d5db07bf788d1f69f950bd2ddf73c550c88b32453e55d15f422","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"3bff499603e7a4d81068e6457b6d94544e865fc70743b9be278f0ea515a10d86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"408ef8388aaa44003433347eae7dd6c7a856b7078ffdd8abfc9a831c782bf6d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"439ff4199ab893ef47b2b24677b3a0712ca10557119868746c09e5ba33753173","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"43fdd0f3790007d3ecddd7c1c777edb218e066a82db7780cdfade1af4e52fdd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"4c8ca200ae34404a7e3bd4c7adc8b9971e6a8732bacff3717a8dfe9f55b7431a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"573441d01bc3a5dc38836ea38903d87ee6c2f25b6c4d93a979010af06a1058f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"5a26c6c58f465808210315cfcafa690d409fe8a03cce1c50af1465b1cb7a8610","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"5ac0c3c598bb23a89a0e63b80193f5a0b989ef8f676e990362a174cef1540ecb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"6281eff7f1eb8279cabe77f9d1f1d2478a97279bf18f2b5e7bf9104f48bb8eae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"6e4f2cee72f249ad623ac333e98b4d5b46ed800feced1b98e734aad6426d4f57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"85b4ba4b35a851104183637b8ccb8be950663bcfce1868008fe118cbe84ca8eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"882cd20f2a88ec92ee19a96ebdb0e7450566486d7063b764f2d65682dcbe6990","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"898be32d8771fbe17057829ab379f352c641969b8778d5c746d951682484e908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"8f18f1765cdb58fe6bdd47ab7382897cef64b6ff0d2819e45a6edfb388427d67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"90c0bc090799cdf6da510003428c2c0909aad37c6a562a21a7d9711e6fac48f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"92e0a2eb8ab9de257cc1b92fbcc3e72f5a75df2a6d7cc7f2881e635a2c3b5f6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"94c267ae0b903b82dad324aa6a1aa031b444202e67e55ac10746b4caf0554318","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"96595f560f0d4e5f6272ce653d963aae178a27545a8e2d6bb342759a707380d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"a7a62f4ae83f39cc2aa5610a1fa39913dbb89c8a7e136bf0b375398a36a4cbe5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"bd0573019a5f2c7a7f36981c8e81650979ee3fa21b9ca74de2f80de6f898a00c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"c455f9856d6fa821cdf932c6b24a1d5b652aacd13e74e0b2a54394592476509f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"c4d86f43a9cb158593bd8be02d96713af2caf1df823e38555f25002469f3a771","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"c58b99118542c64c5e8002b1a143487972b7b81409252899ec6c7d1391fd9e89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"e64f97e86ec58fbf99ca3a6efe60f55c9d315887853d612757b870ea0ef396c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"f8cd0e057dfd4f130119ba94d650784ae8733e232829e5be0bc8cac32b386df0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"ff4a3c1c2ea8ddac1bb3f1cf2f714cd7e5290bccd6c0f8a4734033754899293e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"ffb915c7a519d7b02820c9035036d88977d516b84a237b7e6e8ad2dd9b9161d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"11fafce13cda39efdbda8cbfbabf4f5aa9d728b01c3b8cc3fd705a8eb948f07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"2ba467d1a14629feb97a61c1ba7852b7404d2206ddc7f3762c7e73e411f2a02b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"4540928969b272dea9930016b863cc4715cd1070f913dd0d298a5c42ca07687e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"501206e43f5c4171ac14d29667891202cc96e10a3428f9f639c55ac66c20afba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"d68ec7aad4b4702fa2feadc626a6b2e344a7de36646823c29820e973bf0cd18b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f05f1056654f1d3969661b5157a3329323ee8a790487854e6d8e017502186852","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":20,"runs_affected":10,"failed_runs":10,"run_impact_percent":30.303030303030305,"anchor_occurrences":20,"after_last_push_count":20,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,0,17,1,2],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012916a0\u003e: \n    failed to create NodePool np-dpvlx8: failed waiting for nodepool=\"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\" in resourcegroup=\"rg-nightly-5-0-5jrsqj92ljxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\",\n      \"name\": \"eee4d653-3b34-478d-9851-5151f132ff5e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:23:56.849718476Z\",\n      \"endTime\": \"2026-09-29T00:43:01.619582837Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012916a0\u003e: \n    failed to create NodePool np-dpvlx8: failed waiting for nodepool=\"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\" in resourcegroup=\"rg-nightly-5-0-5jrsqj92ljxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\",\n      \"name\": \"eee4d653-3b34-478d-9851-5151f132ff5e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:23:56.849718476Z\",\n      \"endTime\": \"2026-09-29T00:43:01.619582837Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001873458\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-zwldz5957z9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n      \"name\": \"00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:57:06.99965356Z\",\n      \"endTime\": \"2026-09-29T00:16:13.233773533Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-lwwnr-xksft: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-lwwnr-xksft: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001873458\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-zwldz5957z9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n      \"name\": \"00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:57:06.99965356Z\",\n      \"endTime\": \"2026-09-29T00:16:13.233773533Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-lwwnr-xksft: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-lwwnr-xksft: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z624hhq7js4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"name\": \"c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:15:40.256886606Z\",\n      \"endTime\": \"2026-09-27T02:34:46.354762492Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z624hhq7js4c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"name\": \"c94480ab-33b3-4cd2-b8c1-de35683bd4b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-27T02:15:40.256886606Z\",\n      \"endTime\": \"2026-09-27T02:34:46.354762492Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f7a0y8t5i5r6v6p-np-1-8s278-9tdcn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z624hhq7js4c--managed/providers/Microsoft.Compute/virtualMachines/f7a0y8t5i5r6v6p-np-1-8s278-9tdcn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"79be886538684366411edc9bf39d37cbb253c90860c0f3a74b84b86bf92ed16a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"be520732f5a1c2ae3f44c1f3399ac933aa77be4a8bfb047291c2b5bfc9032219","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104028630299447297","occurred_at":"2026-09-27T02:00:24Z","signature_id":"19018416e82eb03f9b16613ac5e37d0f324be8fff878665862029a9f827bc1f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"089da5e62c30a2a9e3d1a6245182e71a4642a38a698b3b82c0cd96b9913dc8e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"f9d15553519d6f900df80d9635a99e16a0d600a336f2c5c4d29422be77ce186f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"4e0a0620c8831c0f951a548fa2513f2279819e6296978d0d4cf8e21c727d4cec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"dbd0f8e48eac3e4cc622d79cb9d2ec8cbadb4a96fb50d05cb0b2554a97288e9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","signature_id":"5c06e0d910ddc2acb4d5603c5eadfde618b3dd69052efab39a6a1f1a0a93424d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","signature_id":"a76814b56f15ee47998572c134135162901cee1455aa045f2734242c1a906a9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"103961703b8016ef8d2dc0e2f58b7bbb98c3574d1dc3c74e67f1d240fdf03492","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"28839740e2f9c23000e6b0500d8c0408bf8e17372a61541f2937024db1d36025","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"2c1b9ed23893075bc0ec6c4a6b1037e9cca96febebc0b50ef57090534c5cb402","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"65253901d9a0188b19f3893d1b92022170a5abe60d3935d41601fb4c5d5f1b5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"7049cd27c77f3381a35c5d814c9d2733d694154d7c87b77c2714762c51e52aaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","signature_id":"82368bdfff8d1629c3e37cf188eb5b40952905d54948f637fde213abe6a41563","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"3e311527d14456ac6894554b956c239196c7332831ad87223df28d3ee5ad65c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","signature_id":"306b29e323c6a56937a91e8a9b592e4457743d2ec8d8a321540ac9c29f5c5317","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","signature_id":"cb3842f13643a3ede46cd6921339aeccd780d8ffc478c77a73c368f7b7de93b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103690040713416704","occurred_at":"2026-09-26T03:36:04Z","signature_id":"038e89abf06c9fb71bc46c4424eb7ecab72c651acda302bd05745bbec4db2f13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","signature_id":"61e1a0c5120956316d98b8f902f363d01c50acf5c533fe1e2472e0d4bbca8113","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":20,"runs_affected":4,"failed_runs":4,"run_impact_percent":12.121212121212121,"anchor_occurrences":20,"after_last_push_count":25,"also_in":["PROD"],"trend_counts":[0,0,0,5,5,5,0,5],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002a097a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-swk5ggttxwjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002a097a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-swk5ggttxwjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00160fce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hfkhgg4cj7jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00160fce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hfkhgg4cj7jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b459e0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-sf9f9x57dtfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b459e0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-sf9f9x57dtfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"13f4005d5995a342c9b357c20b4fe80d11b6f71b8daf1ca214978ae01677d844","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"3611ce8acc0c465680d47f7d670a7cddaae327fe71dbc3c6d5e7c236e72e2b9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"44cfa63095c47519677515fcbb5edf1f6fef18f8fa9f804adde98dd248b04ee2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"682bebdf23c8807964a6439128835338cd9f440c606c68367c05915320960318","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"eefa53d30dbe7c20530d9bfcbd3b414e69c13274d0f6976f0cd790c7047bfb7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0851a1986905a5b422808848fdfdf27cfb0c40343e7c8b366a189c08d2b9281b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"838796bcc7d4a7577ada2de39969a525ff7d7bafc0d94b7082c7538d5eda55e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"ad2672a3362fafb9d20fe759bb8cd0e50a98d6e39bb8af803f5a3851f16a2e13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"db9eec2083329b01da8d2cda46e73d2a9ef0942d7ceae885ce0bc1271c38f74e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"ed12e18092a6cb363796b8a2d7f31ebc10c5f1a6cfc32b6f300e750016e2970d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"0e7801135f075953518765c1bbbddc68cb14d38ab7ce1effafb5ee7011cd9767","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"4e21bc12c99ecec673a96eb7593881309ac7de9775f283ab767dd467523d44d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"50e75bf1403da3027cc0838aae74e5b39ec954431257679b542bd194fafd38de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"a7b728efe3485e59acc90fa1890af7ee6cdc0e407befeaca1f81b294e8fd1008","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"db14bfab8e3a48be52d9cfa18b18664997f945501b054b544b3db7caf90c40c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"27740ed35f7486949fc1f199355bb1882c9f8cb9081d779fb41653a80527acd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"691dfc6d9fb2291ffc32616264d74d1f43851ef85e826708db94455617cad251","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"7fbe71528d20c91eedbc5d9a15147ed84d81599aec91f81260d0bbd325e1866b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"9c871760b90c87c480ead77cf16197e08922039d3cb3658d834ce5592bb2917a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"c5e7a15d5897753355cc4005fa9bf61ec537390259186723bdfc09da6d61808d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":14,"runs_affected":5,"failed_runs":5,"run_impact_percent":15.151515151515152,"anchor_occurrences":14,"after_last_push_count":18,"also_in":["PROD"],"trend_counts":[0,0,0,3,4,2,1,4],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001432880\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lllf2h\" resourcegroup=\"rg-np-dg-minor-lllf2h-lmftzgmmkgfx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lllf2h-lmftzgmmkgfx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lllf2h/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001432880\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lllf2h\" resourcegroup=\"rg-np-dg-minor-lllf2h-lmftzgmmkgfx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lllf2h-lmftzgmmkgfx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lllf2h/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e6340\u003e: \n    failed to create NodePool np-br2c4x: failed starting nodepool creation \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\" in resourcegroup=\"rg-nightly-4-20-bvth8x7dzstx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-bvth8x7dzstx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-br2c4x/nodePools/np-br2c4x\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e6340\u003e: \n    failed to create NodePool np-br2c4x: failed starting nodepool creation \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\" in resourcegroup=\"rg-nightly-4-20-bvth8x7dzstx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-bvth8x7dzstx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-br2c4x/nodePools/np-br2c4x\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dbaaa0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-grgfk2\" in resourcegroup=\"rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-grgfk2/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dbaaa0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-grgfk2\" in resourcegroup=\"rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-grgfk2/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"6dc5b114f6ac687a7b4c1dd7c710e60fb5ddbfa17d78beb65f14479f54bce3fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"938bb68fc81ec914d6ee9c7b65e3ee227bb2a191b4973a9f265b8fbb7db113af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"cbace1c1515d3a38f062cbe5eb63ec2ea5b5fee3036a914597181616e998cb1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"d15cf8226866bf1b9e3e386a753681c173badd9fd0c5b9b634779ff33a37e3cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"2fe43e4aefe8fad2aa3b31f0b353a06e9223a44a64d2a9d195a8c3bd94ddf6e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"7893b4af94afe3c0281e1d0d5e4a28b92a42f2a0ba01657f8c46a66791bde30e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"83c3157c23d8aad0c7c51357004cced7cba545ae6f2dfec2d5c5e89d40ae1156","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"0cb35b890f581d959b6853f4aea9845c15baa5df736da136cda474135f2e5079","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"7025c972aa4849457e5ddbe4ef9be30261f8964b5b0a0be1395928ce0cdc81ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"8e4966742e7d4d0ad960487d3edf5beaf995ef862658f56dd2fa158c7259a509","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"e5fa85fb92e02ccfc9e63d3a7fab3f53bfabfbab7d6cbda05b9d8eae9f85d1a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"1b2eae294949b5b68cbe342df9ea515c496f1ce1130bf8c19dff2578413bca2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2499142f7ad8e903e0f48a80ca5cf2771b399a595df8551871acaf3736c67691","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"3b28ac3f4fe19a90ee317ea1737bd43f097cfc518ad408fd1fd876a861e94ad9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":7,"runs_affected":5,"failed_runs":5,"run_impact_percent":15.151515151515152,"anchor_occurrences":7,"after_last_push_count":8,"also_in":["PROD"],"trend_counts":[0,0,0,1,1,2,2,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d235a8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d235a8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011eadb0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011eadb0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-27-005925, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001257740\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001257740\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-09-13-024541, targetMinor=4.21.0-0.nightly-multi-2026-08-16-064216) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-16-064216\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","signature_id":"91d27f5f51891910ea04c7a1edc0380a08aadc419bba96bc422aa6887c894e1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"48061177828d591484921b302c81a59f13ae276493f33f536f8259e7bfc70589","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"9d3c0ca51b7611f46d699909444e88f85d187197f0007cf922f659d4688bf7f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"0fb83beb0f702d994cf5a20ec546309e606192d5d552ebb8972f4ca02a52a5a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","signature_id":"1b6a70986e38df8901401ab55acb5556c4468c54133387e22f1e45a710ed2281","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"a7c8a4561bc6869b3844c32d80edf46aa9a5ca5311095d7f67b3d5dce4de9b9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"bb492d03ce36d043a7737e16613de6f3e8eaf5e4ffe34f36ac441cff010efa9a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.090909090909092,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["DEV","INT"],"trend_counts":[0,0,0,0,3,1,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-26T02:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n      \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n      \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n      \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n      \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feb180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jmvth8gmbtdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\",\n      \"name\": \"c689250c-de76-4270-9b21-5b67d697be13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:52:06.978905427Z\",\n      \"endTime\": \"2026-09-25T17:11:15.434963979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000feb180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-jmvth8gmbtdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c689250c-de76-4270-9b21-5b67d697be13\",\n      \"name\": \"c689250c-de76-4270-9b21-5b67d697be13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T16:52:06.978905427Z\",\n      \"endTime\": \"2026-09-25T17:11:15.434963979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-jmvth8gmbtdp--managed/providers/Microsoft.Compute/virtualMachines/y8x7q9b3i3q1x0q-np-1-c8cxv-t7pf8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e30fe0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zw4cd4\" in resourcegroup=\"rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\",\n      \"name\": \"995a0744-273b-428e-a7c0-5da145da2012\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T17:47:45.849510742Z\",\n      \"endTime\": \"2026-09-25T18:06:46.935940984Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e30fe0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zw4cd4\" in resourcegroup=\"rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/995a0744-273b-428e-a7c0-5da145da2012\",\n      \"name\": \"995a0744-273b-428e-a7c0-5da145da2012\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T17:47:45.849510742Z\",\n      \"endTime\": \"2026-09-25T18:06:46.935940984Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-zw4cd4-2qsd7bvk6zfl-np-upgrade-zw-63ddcd9b/providers/Microsoft.Compute/virtualMachines/p6t3i7s0z2k5u2p-npupgrade-4-20-8tr6b-nknqd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","signature_id":"f981c1008c821f13aae9959c202473fe9605a11d0fdc588a129f3cedf979334e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103519896854859776","occurred_at":"2026-09-25T16:19:59Z","signature_id":"d94abe80bfe3331c4bae8c789b9cd27e17b79e88882a8166c69ed09fb5c17b41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103519896854859776","occurred_at":"2026-09-25T16:19:59Z","signature_id":"e625f6b2928df703a3e0c754e899a049523cbde56debbcd37376834e602b6278","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","signature_id":"d0a53dcf7d7bdcb9735560a86453bcca992d3fce7123e13092abd431e1b00745","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.090909090909092,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["PROD"],"trend_counts":[0,0,0,1,2,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-25T04:43:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxn248k7 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hwjzhdrfqdml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000973dd0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxn248k7 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hwjzhdrfqdml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000973dd0\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4lqltr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gvcrd9xg4vzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0008ace00\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4lqltr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gvcrd9xg4vzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0008ace00\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000d18d20\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000d18d20\u003e: \n    storage account log verification failed\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","signature_id":"e3d755946357e33c2f103362fb47a38b0d13a5be02426a90a6d3deb944030095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103303533246812160","occurred_at":"2026-09-25T02:00:14Z","signature_id":"889564d15d770295c243009ba9604dcfbcd6afb5d6a4b37657d92965a6c4aa98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2138dc55899dce5641331077f3e289aee19252fb625b5f07c0d419ec286f1599","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.0606060606060606,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,3,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-26T21:14:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014c6168\u003e: \n    resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n      \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n      \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014c6168\u003e: \n    resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n      \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n      \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c140c0\u003e: \n    resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n      \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n      \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c140c0\u003e: \n    resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n      \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n      \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a00138\u003e: \n    resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n      \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n      \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a00138\u003e: \n    resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n      \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n      \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","signature_id":"be9b62e6f592acd1637d57bc1488a35a561ba98109cf2972d839ae6a04fc56ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"66487e377bb5836614b65c245bd64eda43f31dc7d151c0a386ac9d9a8cd2fbe9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","signature_id":"b827dee0a9cbc7d4f17c48bae9a3424d764e85fb3cd960112ea93b2d8250799d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b0e52789a9254af02bf84a6572690557e6e470276175202a83ecb44c6d1cf3d1","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d523a0d2817fdbba88731dd1892c07aa03075f35ab21021c384553c8aa80da9e","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T23:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n      \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n      \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"ccb025795b93e958b7aae9d388bb9cdb57b9a396064e500aff59a625a9f0e777","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c071429a287be0f254f4f0e70d2ef61574322b99e904b0a2496ef6afa776b8ab","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-tklhx4zzdx6h/cluster-nightly-5-1-vjz6jm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e262a0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vjz6jm: failed waiting for cluster=\"cluster-nightly-5-1-vjz6jm\" in resourcegroup=\"rg-nightly-5-1-tklhx4zzdx6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\",\n      \"name\": \"5f9063e9-f978-4377-b5d9-c9222622088e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T00:24:56.11201607Z\",\n      \"endTime\": \"2026-09-26T00:54:00.860881838Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-tklhx4zzdx6h/cluster-nightly-5-1-vjz6jm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e262a0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vjz6jm: failed waiting for cluster=\"cluster-nightly-5-1-vjz6jm\" in resourcegroup=\"rg-nightly-5-1-tklhx4zzdx6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9063e9-f978-4377-b5d9-c9222622088e\",\n      \"name\": \"5f9063e9-f978-4377-b5d9-c9222622088e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-26T00:24:56.11201607Z\",\n      \"endTime\": \"2026-09-26T00:54:00.860881838Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103620631026208768","occurred_at":"2026-09-25T23:00:16Z","signature_id":"4c4499c70bf7e7fbd7efdcba08b18c93c68baac4fc5629cdcc576459752fe59d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"333a2f1360dd6224035baf136a6281579fb567d23bfe26dee12ef543a72e7859","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want \u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T21:51:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015ba9f0\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2rtwrg4pjxl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\",\n      \"name\": \"88abacf9-a5e0-4248-b32c-83d6cd062714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T22:23:02.156910064Z\",\n      \"endTime\": \"2026-09-25T22:42:06.143243532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015ba9f0\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2rtwrg4pjxl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/88abacf9-a5e0-4248-b32c-83d6cd062714\",\n      \"name\": \"88abacf9-a5e0-4248-b32c-83d6cd062714\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T22:23:02.156910064Z\",\n      \"endTime\": \"2026-09-25T22:42:06.143243532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2klmb-6glqn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-2rtwrg4pjxl2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2klmb-6glqn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103603313239920640","occurred_at":"2026-09-25T21:51:27Z","signature_id":"ea79c06297d62c705cc8405c02627178ecb3397453f3b4a80d1e574e28dc5c93","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"41dadc374b6fe2625e6ee7acdf8fbc9a991cbd05173905d55b61bd73224d1bb8","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T23:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142c760\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n      \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142c760\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n      \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2c832c99908ab7e4af98af64b3ea848748c2d5da97c25a86fe752084f15d42f0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001152660\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-c95x5l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n      \"name\": \"5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T22:54:48.090437049Z\",\n      \"endTime\": \"2026-09-28T23:18:56.801422937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t49njjvu97eroh1p7j9jssq535c6590 still exists (deletion dispatched at 2026-09-28T22:54:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001152660\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-c95x5l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n      \"name\": \"5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T22:54:48.090437049Z\",\n      \"endTime\": \"2026-09-28T23:18:56.801422937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t49njjvu97eroh1p7j9jssq535c6590 still exists (deletion dispatched at 2026-09-28T22:54:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2104687818398765056","occurred_at":"2026-09-28T21:40:53Z","signature_id":"b9887fb5f4ff622cd6b40c348a1b64080f5877eb981c863198ae3705d8ec506f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-24T23:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n      \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n      \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"b1a2325b06cdb9a45a78b9f23c355fd6276e541cbba1065d34722c4530f70a01","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"68cf65637733da6389b66f0d98a7fd646026fde9e15cc697ce29959bae80ec43","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-24T20:07:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"847f7b17102dc7090d588a9d72db0f68b89cbbd60c61d629ca5d29b7d31645cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0d6238e39baae339fded79308f27dee513cc7e737ebdf99b36269c7a4d445524","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-25T04:43:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000b17180\u003e: \n    Operation cannot be fulfilled on nodes \"d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000b17180\u003e: \n    Operation cannot be fulfilled on nodes \"d7p3x3q5p9p9x4z-cordon-np-7bdwq-kf6d6\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103344681369735168","occurred_at":"2026-09-25T04:43:44Z","signature_id":"3f9b45040bd18db3decbda1404a3ed874d541ef97cff863c3152e40ed839bf48","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104345445907042304","occurred_at":"2026-09-27T23:00:25Z","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"8d719d468a5fb10bc89ca62a043bd8700c617674d20a67919750ac1d3434857f","failure_pattern":"failed to create HCP cluster \u003ccluster\u003e: failed starting cluster creation \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": Put \"\u003curl\u003e\": stream error: stream ID 37; CANCEL; received from peer","search_query":"failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.0303030303030303,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-09-21..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":1,"prior_last_seen_at":"2026-09-22T02:00:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-6bkkt2 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013940e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-6bkkt2 with version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013940e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6bkkt2: failed starting cluster creation \"np-version-upgrade-cluster-6bkkt2\" in resourcegroup=\"rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8\": Put \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-6bkkt2-m96mr2dxx5b8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-6bkkt2?api-version=2024-06-10-preview\": stream error: stream ID 37; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102216478680944640","occurred_at":"2026-09-22T02:00:40Z","signature_id":"483d93133052646e81580234624852295a4a3911c14ae9d5bfcd4ed0a65bbd82","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}