{"meta":{"start_date":"2026-06-05","end_date":"2026-06-05","start_at":"2026-06-05T00:00:00Z","end_at":"2026-06-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:01:44Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":5,"raw_occurrences":109,"matched_occurrences":109,"runs_affected":5},"rows":[{"environment":"int","failure_pattern_id":"0699c0ffcc0c61c54bbc3353e8c597e20175b980240380c7809f7916d9283573","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":66,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":66,"after_last_push_count":66,"trend_counts":[0,0,0,0,0,0,66],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should update NodeDrainTimeoutMinutes and upgrade a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22320\u003e: \n    failed starting cluster creation \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-96clml\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-96clml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22320\u003e: \n    failed starting cluster creation \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-96clml\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-96clml/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94a00\u003e: \n    failed to create HCP cluster np-dg-minor-588mm6: failed starting cluster creation \"np-dg-minor-588mm6\" in resourcegroup=\"rg-np-dg-minor-588mm6-sldfjv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-588mm6-sldfjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-588mm6\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94a00\u003e: \n    failed to create HCP cluster np-dg-minor-588mm6: failed starting cluster creation \"np-dg-minor-588mm6\" in resourcegroup=\"rg-np-dg-minor-588mm6-sldfjv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-588mm6-sldfjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-588mm6\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ebe2c0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v6j2dx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v6j2dx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ebe2c0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v6j2dx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v6j2dx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"043de65f262e55ba0002209ba94c992dd357e8cc942e2143a3975517d1d1e8b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"076d9b66c87a01b8395302ada00a635ec67d6623e6e5715f1629a5b6b931c668","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"0e0b69ea03b08142b6d1854e03dedf81defaa1184e30a3aa0835c6ba501d87d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"0f40823828ffef56cf981058523c8cea58000d751e826a41b0a3e61715af94ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"14d2237dd0c131350f247f686174e57fda09f64503e6de33c70abd8e03bcfe7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"1677abd9917463b0f0a49dbd254ef7d3d0ef08a545d65bc19689e33a0b42dfc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"233cb4b78b7e77c8627e1c00320687c63146beed931cecd37c209f8e26990054","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"23f0f367a8a1e8d2a8545f0d6744d5677d784aac41619418b7b3efdb78764611","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"2723f2da20d43babc204223f6672b81058bf39c3f7e33b765f52c2fdeee2ff28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"2c2a6b8d2f20111bbdd2fc8012da921745fdf641986bd7366dc4cce709751e04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"31faed8ee4739c3d27b2c26c54e2017b47b74afbd4b931a31e6d0dda2b5488a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"32934b4f1285c9ed6d89660dc2528972159c868c285a48ba1ba03e730c8a1545","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"39262ab87063703f7297e4b3f22b72a75d05e9231449de8386f7295906515a06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"4690c3953085a8e206acd70b98bbb0b9efd4a184a90c6962392fbaf830024788","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"4b326db2d6015f5c74253123d8e45f8419fad244b283e52fbe8e1a281ea368b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"5c8599f8d6e0c766722979f4ee72502a3d32e9aa8370fa7a8fa76ca986bde75d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"5c8637546597fb49a94597355d40ed052c5641a44c450ee0641e7d4c6592550c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"5e833b24e9818fba226661b4888a2a5de796f2ba9e7cea4473f7723085580b56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"6716ab582053c98188e75456eff1698739eb2c238ac8ef9cee49c2f59e108919","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"6900e15d0d1fb9eeaecddaec2f7dfc61470e2532c98a1afa53031c10e4a40273","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"69713dca79c58a15723d16740ae1abd81784e3c090b495201839dd4911263b75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"80847fb6bb1a0b295afaca459d8ece0015bad80042b387f3cfa819308cd806ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"82e1a395eb074f16174f68f107fbdd2a18ab3d957b9baa3b82c034f579ed1079","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"89867cb54e13bd074a27530adf6c9f4ebb521aece262cc257052a94f0e938327","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"8ae9b7d2aad4fd54ef0a61dcc95cb7076809d7448b45aa20a03cf14be421baa9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"8e68e6727fe9a98f97edd6dbd8a485bed54d425f76cec2025ec030fedbe031ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"9149abd98bf4087dfe025f0e6adfd2f47a7b70c44dbd75885b0a5e82a8480296","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"92adcdc5e32163249bafdf80dec8d02ce9fec2a3d392e42af187ce66bc66799d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"944a227a9fe863b76e828a2fa31ff86f826f05c576d8a71e08802286a99f67d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"99a641bba2b62e304e50a1d8ab772a8d540a07cce5741cda407b456b3e144768","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"9dd18d292a2cce5e08c1eb44eac084a65c2a5da6129824918259e925b5021ca5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"a44962414699d9a3cb197ea0ef068b32997e3e7b879965e35a4a6f86722ab309","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"a9c7735aafc37d3f8216c571c99b72b5f3e32a39f9685487fc793e2b3563c420","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"aaf593875e56e20ffddd7391b75fc6e2aa5482b1a2c819c1f54da1d3f60aac34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"c4e3a8154d28bf2801d738df7edf90bc7e390af4fb2001db3f041bcf0b32f9a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"c7e722b063c3231c7dc9d674fccd324413da1c75125a91169950052bb89f5f02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"d2873bd932fbba543303f4d4f45049770986b778a1a0bf4e6f8ed6041a6442c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"d96b2cfc5ce255505c80738533f34fcc83c150eb98eb467e65215fb1dde5b361","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"d9b79201d7875192ed5a9e4b3e645a1f533990334e2b64bda24645bd39c31c8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"de12609547c4c1844aca833146b9926c9c2843ff58b515f434d1ce5bedebe82c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"e0e58553e7087b04fab2b547e1e4813f2f61ff8bb892603e01d37e10a25da5a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"e5aa5083cab51d747c6539f873f8853160b8c7433317d3aa106b42eee99315ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"ed077df162f58b9da1347a5392fcf8619f58511eb086c8cbd8af36fb5bb916c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"edb5d62e434c0e31ea99f0c549d3b1b4598cc57a47a616371b3ce971cd9e37fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"efc42a887fdee6cd988b10ea640671e803fd83aab2bb39b0c64af509c4c83eea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"f4d31638f9554768a20fea8b6b274d41eaa21eaee4f10651896f07b9da09e597","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"f624d75f7c7d3ee0ccd3a9d019b193179c025cfd747af98e9120bdaefe35a698","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"243ee0d5df644dbf1f5d07686d492e612d0a3db2b2622285fd4f1d684673a5e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"43441a7992f631e314c60d2befe8cb688198d83ed0adb1b866eccb050fd622e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"51753b17544b0d0b7ed819d43d6c25d77ea053c302a10e84d0eb201ddd5379e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"63876759985633bc5bfb9998bd6743ea2d53d18eb412a99d4c5dbf798c29cca8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"6e52d097da65931772cca79781ec9c4e6b9f4717b36251f9db839e3ffb56a3f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"7302f670486f1a1401164f60d8e4a794bf9cbd504e9c3aaea7cca974a2e62283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"77c08b068f47f07809ecc59a7e8c7f728fe5798ffec946b78ca9e79dedb38647","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"9ead73fea3688646bfc6055572dedaca172291c53bedde5189860d190bad5c63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"a8372844d259797763e20d4396ac706c4c107b580cd357b55de2d1830fa4fa83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"b5cff975b39eff770d353eb0525f1355477e159bf55d74d6348b191de5473fbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"bc3399c61aa624ed167438d2fb1a112d063ae2dd7ccbaeda3f01f70ecf95a682","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"cadbb8de9cfc1f694477f79c6677aa17dca91695175cac8fd3777c83099ca896","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"ce815e98616efab9fbde9371ed5e305edbef230d9a7993d3405f8ebd68588895","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"d10a54fcee7832c2d8d8469399e63832330407c79860a351c2046b3c0711fb21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"e1684b16ca1195bcfef4fecd87aadc97d3c1de7b204dbbfd259632a6eb8f1002","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"e20c18f85ebab292604eade54fc6da3a1f704767aaa7c173e0ee6e4c7c4d5865","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"e7248343df76c9f1c8b222e06284443e9d1890054aecb8f3bc37f8a579df2663","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"f5af3cd54b639b191be97c6cf17b30cca166fb2ac5e93e474a872272671a1c13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"fceda00b6261a82265652ed410daa8a86833c63fbeaa1031f23ae3d51b6630cf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-ftnjbl, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-ftnjbl in resource group rg-candidate-4-20-24zngp, error: failed waiting for cluster=\"cluster-candidate-4-20-ftnjbl\" in resourcegroup=\"rg-candidate-4-20-24zngp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-ftnjbl in resource group rg-candidate-4-20-24zngp, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":35,"runs_affected":4,"failed_runs":4,"run_impact_percent":66.66666666666667,"anchor_occurrences":35,"after_last_push_count":104,"trend_counts":[0,0,0,7,4,7,35],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-04","2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":34,"prior_last_seen_at":"2026-05-28T06:56:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-gfrtm7/cluster-nightly-4-23-8nml4t should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001117740\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-8nml4t\" in resourcegroup=\"rg-nightly-4-23-gfrtm7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-8nml4t in resource group rg-nightly-4-23-gfrtm7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-gfrtm7/cluster-nightly-4-23-8nml4t should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001117740\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-8nml4t\" in resourcegroup=\"rg-nightly-4-23-gfrtm7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-8nml4t in resource group rg-nightly-4-23-gfrtm7, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008530e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4hklqq, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4hklqq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4hklqq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008530e0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4hklqq, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4hklqq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-4hklqq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-20-nwwt4s\" with version 4.20.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001572780\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-nwwt4s, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-nwwt4s in resource group rg-zstream-upgrade-4-20-gx9b6p, error: failed waiting for cluster=\"cluster-zstream-4-20-nwwt4s\" in resourcegroup=\"rg-zstream-upgrade-4-20-gx9b6p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-nwwt4s in resource group rg-zstream-upgrade-4-20-gx9b6p, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-20-nwwt4s\" with version 4.20.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001572780\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-nwwt4s, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-nwwt4s in resource group rg-zstream-upgrade-4-20-gx9b6p, error: failed waiting for cluster=\"cluster-zstream-4-20-nwwt4s\" in resourcegroup=\"rg-zstream-upgrade-4-20-gx9b6p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-nwwt4s in resource group rg-zstream-upgrade-4-20-gx9b6p, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"0b0be31e4e2dc3815086975d58294aa65f16b527fce764dc9e95ab2b09694b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"1059060a9fd69f1a3bb696683bd2bc685e1dca5598ed8e3835ac310d8c085102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"18e2f4b78996fd516be90f5a0c4575ea98be7ef62913ddbf13555fc7a6336919","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"1d5c5eb7c141042c43d38c9503204400763198d83941c10f70a504c0d4907587","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"21a926cacabc842ce6a821a94fecf58aae7a8778f85a50d0ccc582c057b36b82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"224006a24b09888dda8781f3bda0d4a830b1e448e477119f2c54b77f0d72e3a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"238dd091b32e6688c39de0a2f58d5b5e3db6346404c72d5b130447dce11ef071","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"3430a64ce7081a59465698234048a6af62668c8d5cf754460c298fef9c68b6c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"4d7eb694c0e0fe8a59e0c2feb44adb0fee9d1db46a6e1df59e6adc88c1aa2ac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"6dd33a91e2e41bbc45c1d2c7416fe3d0cf33f359fbd6077d73f4dce1138b96ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"7b45c109681c1a30c10c646efe6f988ccc129ec5ffef9817047a001677640d0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"818586eac67f180f272fbf82ee51d05cb9f2a8f11026711368f6340c96c86c02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"84be83f62b8834633c63de63e7593ab043653e53a53996520c7a66723dccddd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"99f7721f18efd83b8c423bf9dfaece67276b4cc3055ef7a4bdcfc5396b3bfdd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"9de2a7de94af356b7f8480bd9e820e6554253d27d43b914f669073455eb5b1fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"a53f8e800712c7113d5f0e3ce0c8c2ed14311bfabe08fea94db9993b28f73c39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"b16509681ef82209e76314f315a1763dab3401de4d9bb3ce6c9cb66d1aaf3013","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"b3a650520991190bef27f2c027ae6b1180cd9c1d8bbc5919b9185bf4fffaac15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"d04be9f06bf20360a0f2c798d6b014730fb2c775daea152e038d6bf0b81d6e68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"d365036e78e78fbfcf0c8adde6ddfd01f0bd63573f51a09cba887b75295c9e63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"d543cb4e5dfcd12369685a738698cd7335ce8dc328ffd89154f8ea15c1f64308","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"e42d686cffc84ad66336436364b4e3f91afa375a96dec43218fc2137d633827d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"eb6cb53a9ff94c626c46564e3609c99fdc91b216953c26e3afa231faf279366b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"ee216bfc3e748342b42df8bcb6daaa1908b8b932847ba529453f70e9804cce14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"f428933d606f254bb8974ac1751c34b1fcedd30e4f20aaeed6a64ed207c4ebf7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"f67c4229c39ef1bb1875b75805a1142efb32cc0a5cd4ad481124b4a08ac5f389","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"f857f5a7e16b4cb90ec43adf7c1e10e27d0ed83e2589f5299b870294b17fb2cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062854507762552832","occurred_at":"2026-06-05T11:10:03Z","signature_id":"a53ccc6617de67a27c50d77e9e2eb80781cb2246ebba97f6510222eb5882b394","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062854507762552832","occurred_at":"2026-06-05T11:10:03Z","signature_id":"b31c5be33b90183fbb092151ed5d94b4c94075a095973d7f9322e1d3c99df9ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"7aab48828bf8401af54874e60645dbf43181e5e548b0fd69eca1a1373b93a683","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"875a28e2926a36e706460c87c8ece490a0cd1f5023b3673be3a4426ea9b9d639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"db27eb8ddc554471c1cdca2400dadff655cd31cf14342584571ea258ce9e9e5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"df77de235cc90b472bd5717900baef2ae97ab963fd756734489a8eb9f69c2743","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"fbdaffefbe869a6b8bb80d8e8b5341382d9543315d42130a6175277b8d4ef2ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062752332898111488","occurred_at":"2026-06-05T04:24:14Z","signature_id":"59d9776522ccfa3a2a15053867ca82c9668eecda2930082eac08bdda3448b944","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8be64044d6c01fd1b4bb536c29c4b3014256d4589d1fa91f7525668e9f395612","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":2,"prior_last_seen_at":"2026-05-07T18:43:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:158]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpc46s\" with install version 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c420\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-tpc46s: failed starting cluster creation \"cp-ystream-upgrade-4-22-tpc46s\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sgvlxn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-sgvlxn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tpc46s\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:158]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpc46s\" with install version 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c420\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-tpc46s: failed starting cluster creation \"cp-ystream-upgrade-4-22-tpc46s\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sgvlxn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-sgvlxn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tpc46s\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8c400\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ch65j5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-ch65j5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8c400\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-ch65j5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-ch65j5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"687905ff6dba384afec4299db647dae29e62a1fcbe97611dddc93fd72182849b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2063018234914803712","occurred_at":"2026-06-05T22:00:45Z","signature_id":"afa1ca6851368d2d6400aa27067ad69564033c33c37415e1af4b4c869d8537c4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-b8wwfb\" in resourcegroup=\"rg-zstream-upgrade-4-22-2svbrp\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":13,"prior_last_seen_at":"2026-05-27T03:09:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001958510\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-b8wwfb\" in resourcegroup=\"rg-zstream-upgrade-4-22-2svbrp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001958510\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-b8wwfb\" in resourcegroup=\"rg-zstream-upgrade-4-22-2svbrp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001912150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rcs9vp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001912150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rcs9vp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"79bebb08f9ac41d1c95d84280e4032ccdef75ad64d260cbddb874c707e249509","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"a5c00d43f0eb57b62baa3a783bd99d29ce15ef76452b179849678f0346a291e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8c37aeb4c0019ca0b37137c104e18d9445d8f58428c982441fd9abc11ccb61b3","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":6,"prior_last_seen_at":"2026-05-07T06:18:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"7808e0d17261c46203073385b1bf48fa70b107abdef9661b60e820b1b8327817","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"cf6ceb1f4deb82bd4d009ce82003185af5aa727442083111220a23e5179d2b34","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"7b311f6b09c3bb8e223e61a1567cbba9f44bc8fab49954456fd9328e616945db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ad90ebbc68bc07360ec9ef5bafada3391fe2b5ecd8072204e0d092215ba0741e","failure_pattern":"failed to poll HCP cluster \u003ccluster\u003e to completion","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:143]: failed to poll HCP cluster deletion to completion\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:143]: failed to poll HCP cluster deletion to completion\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062796151345647616","occurred_at":"2026-06-05T07:18:21Z","signature_id":"02404d74c43a8c3f4c253bacabd942602f0d55d6b46f80b14bfea8a0b82c07ab","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6f244a8f6481578e9e599608418cb8c1a6147f7fdc0ffdf1287c57e58f19de29","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-30..2026-06-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062969998250348544","occurred_at":"2026-06-05T18:49:10Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}