{"meta":{"start_date":"2026-07-11","end_date":"2026-07-11","start_at":"2026-07-11T00:00:00Z","end_at":"2026-07-12T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T06:13:21Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":49,"matched_occurrences":49,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-rj8hkh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-rj8hkh in resource group rg-candidate-4-20-p5vv4f, error: failed waiting for cluster=\"cluster-candidate-4-20-rj8hkh\" in resourcegroup=\"rg-candidate-4-20-p5vv4f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-rj8hkh in resource group rg-candidate-4-20-p5vv4f, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":46,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":46,"after_last_push_count":46,"trend_counts":[0,0,0,0,0,0,46],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":69,"prior_last_seen_at":"2026-07-04T02:00:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-v2vs79\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce48a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-v2vs79, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: failed waiting for cluster=\"cluster-zstream-4-19-v2vs79\" in resourcegroup=\"rg-zstream-upgrade-4-19-j968pf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-v2vs79\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce48a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-v2vs79, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: failed waiting for cluster=\"cluster-zstream-4-19-v2vs79\" in resourcegroup=\"rg-zstream-upgrade-4-19-j968pf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-v2vs79 in resource group rg-zstream-upgrade-4-19-j968pf, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e2420\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-kn28ct\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e2420\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-kn28ct\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-kn28ct, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-ndvw6g\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a07e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-ndvw6g, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: failed waiting for cluster=\"cluster-zstream-4-20-ndvw6g\" in resourcegroup=\"rg-zstream-upgrade-4-20-9crtsh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-ndvw6g\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a07e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-ndvw6g, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: failed waiting for cluster=\"cluster-zstream-4-20-ndvw6g\" in resourcegroup=\"rg-zstream-upgrade-4-20-9crtsh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-ndvw6g in resource group rg-zstream-upgrade-4-20-9crtsh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"295d0ae373f787db7cc03ae271bcd03f44425d98e5b3f6ee496841e90aa0fe28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"4203a2f72ed0c06e15e65d91d1538c714beb38fe96064eb4470ebdbb4bf0f7cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"57f3aae6edcb103ecb4092a0542b513cd2011702b1b90bd88d3b577f5cfb1110","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"5d084c3f3c606aeca0e2c7561717fb59ee835f9af0626324fed12dd974b1ffe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2076079208907411456","occurred_at":"2026-07-11T23:00:29Z","signature_id":"fe83eb2e1faf35ccc0d8b9bef9c33b9bab9aefed6291e2bd9c92efc646f04523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"08318022d91a1ebe9569520821c87125282eaacc8851c8ca08ea8834f0bcb0b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"116964d308f699f4a9b204561316f4987e8e395c1760bea603cc3772dc7cad50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"119c4e0b6d0cd0db96e1003b9c5342c2a3f068f6ecad46837ad2686c0da7f099","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"14ef28aa9650b0f3a94e1b316a686e927de159bb8be9dc207bf5914e64f45ff4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"1f4f602d17a576b45b6b754dcbd04266c44363fc4bb256b43a4a538de5b874b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"23f8aedf2ed982edb9355aec6a186763bec349d317da50d2f0e361d34bdab7ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"26f70fbec1d9c114069fdef5d2735b49df750d6d95d6493d17acd9a8899eee72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"2987c8f4c655ed52d0adcc0f710f0da9b23f651ce5736eda54a5fac6eff6af56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3020355086f3ca721d4c918bc7526f2e0d4aa02130d0508ea39bea2e5204e8a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3096a8feded2ff270a7f17e3169e696f90f7495aacd5cb195ae4f78921923e42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"3bc6fe6265f3e9c776f48938f094bdee22eed66adf8f5a0f57cdf80e8501c542","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"40f98a8fe2ee5769f2f3fe657a600eba6f90c74c363bb5bb349a9ae73cc7816e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"410b9389ac03d165130e802d8c3cdfaec8546dee26ac9e77090a402cc716e054","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"4d15f4dfdd2e58e51237555b35bc63600ea314e59343dbcfe31341db234aa859","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"533cbac3ec6be96bb5072a542410db60f8aab4366bfbbff9e6ab5b55066c1d5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"5349761d55e1d3b4f54fb24a03e78f21149730ff6a3bcb25874bcd358b26c885","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"55612e7e60aa5b214a343ec878b11b799f3ad15228bb762f6f2d42f273d58f8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"636460635c5bc4286ba2d967ca22e0e23b6b0e047c29f5fb9c0460bd5027ab47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6a70d6a102c3d67539b0a85dd97ef0f7d4875dfd553445f82797dd99de495762","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6cc23d31934165cf34dc32009b6ff0c069693722df2dd1feea55f38c801b2288","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6d6808d057091e810fbd2cffa6748da09cf1ab8fcb9cd1e21499cf0bb9d6d89f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"6d6fba57278922b4cd3d4ae33b1a3dccd60667684616d769ac5698eabd1775cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"71fa4daeddf941d959f8a5e7436bfa6ae16ea764292a86d6a5ddf60b132b49ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"758aaae83e0a9fd755bb48e054201f5460f0f19cc2e94df9eab93a2beada0d25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"793c9d44ab88946c370b5371d83289174c3890e70a4495861563c7efa748f749","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"84e37b2ada7bf4aacc5da593356f968ad671fcbad815be3ff029c179d4e70411","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"876a3197e143a50d5b65d13c8c09799475bb5402159370b150dc52035d542657","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"98848efa726d399117c2fd38aefbdafdc74561ebaf221abd6f86c0454095723a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"995e2333d4b71119df0c43ad4e61198a20ea720f67a0f3e337581fd90baff684","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"9aecf61eb5a9f9a975e5aa14f39a90bdfc38ac523058bcae8f9bf4dad89c51df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"a22a53e8f9a20a64c491333b1486075bd0f09e83eb105d01787348d84d3680f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"adcb37889e62838134af124399c06ce3a8853c539af8b797cd7f8bbc79adb5b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b5bbd507925ef8880bb5f5324eca49f058e039e9ba23da80117b4b33696baf12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b5ecba9dbcb5f6799c7849b3c68905933a4a66a18831e4a23305eb62109932e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b775a05d51c34270a80137e04f37d044984f99e0b585f5c6325dd060a0445c5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"b94b7ba3fe669c5bc9e337f91cd0851ce2b2431a762139f95992782056ec5752","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"bb881a23506bae75af29bb108e4b187d3c077919067f25560736d3f9fbfdbac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"c5e8457b43f70984bf5e02b577526f9f57ca7abbc1b2e88a15d8e6402b2d9b54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"dcc962ab743e58b2b00783347a59e279647df2aea8105b5c97d8cb8e7458de06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"df9d44464abdc3c3a029944c2955eff3ef6d3edb2e2a6f7d195aac8b5bbf1e81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"fa3f9bf16be572d3463b45f36846100999ac8349178ce130103c33e88b7f8cde","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,0,1,0,1,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":12,"prior_last_seen_at":"2026-06-23T05:14:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.001s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"7808e0d17261c46203073385b1bf48fa70b107abdef9661b60e820b1b8327817","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":13,"prior_last_seen_at":"2026-06-25T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"ccee34282dc21302f7fa32a6cba8c9d0ba5e20e69efec394a8bdd9c07175ee39","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79232f7fe5f1a65f3af30b7ba05ac2a1643a1a699ce682f3bc11178c03f0cbf4","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-06-22"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-23T05:14:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2075762116744187904","occurred_at":"2026-07-11T02:00:29Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}