{"meta":{"date":"2026-06-25","timezone":"UTC","generated_at":"2026-08-28T19:07:22Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-25T23:00:08Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0591276efeeee9e125cb716775fac103db1372aed8e89d7eab5ca5f7c877f96a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"960ea95ebab045129c0a7d741aa5b176969f6caca70d2a1c494892a80e7326cd","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9084f2110597d124617021e3256fb00df9382b7a2ddf46ca88bfb51004cde916","failure_pattern":"failed to wait for second cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded"}},{"row_id":"14d17bc2c2730840b1eb1fe2b9955449d1c47677ec111b25662c52d4fff5ca38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"f89b477517d598a63b48e828894b9a939c56bec116fb50f1bc37839313d3834c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-thm68z/cluster-nightly-5-0-r4s4xw should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005d2900\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-thm68z/cluster-nightly-5-0-r4s4xw should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005d2900\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded"}},{"row_id":"4cdd7137b379f208f9f8617fbb5cd819945d71fbb4e9f590c156cac6ce5dd7fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"1ce3a33867651f0f37bf41c0a1fe91456a7157672a195c0c514aba47f43da05e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016083c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wf8drp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wf8drp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016083c0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wf8drp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wf8drp, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded"}},{"row_id":"6fe344adcee63dd13a95c5d1cbdc64e767a4d32b1a9824e5be04b980e3a8b137","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"b19cc36910d41d4a1b138355e9ff5cf2fa93dd5b8f578036f3c7ef3e12f3557b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016351d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nvgmm7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-nvgmm7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016351d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-nvgmm7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-nvgmm7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded"}},{"row_id":"fdff17baa182ed338f614708fb93298d890d7ff29bc37d0d07c23e581808825b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2070280913844965376","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"8d91b2adc4fb8c3b24f642854a4c21503092a450afa8d61186befc6b2bc18c5a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","9084f2110597d124617021e3256fb00df9382b7a2ddf46ca88bfb51004cde916"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2069963821643993088","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-25T02:00:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c91c2147e9fe54ab464f58938e2dd0ec133bc5d55ead1e3547a10e1760265ca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2069963821643993088","occurred_at":"2026-06-25T02:00:07Z","failed_at":"e2e","signature_id":"638bb71f0d83e07cbcb8f55179bf16099ec03e152f19a3737ceeb3237329e2eb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-21-49s7bd\" with version 4.21.0-rc.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00077cba0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-49s7bd, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-49s7bd in resource group rg-zstream-upgrade-4-21-rj5gpm, error: failed waiting for cluster=\"cluster-zstream-4-21-49s7bd\" in resourcegroup=\"rg-zstream-upgrade-4-21-rj5gpm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-49s7bd in resource group rg-zstream-upgrade-4-21-rj5gpm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-21-49s7bd\" with version 4.21.0-rc.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00077cba0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-49s7bd, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-49s7bd in resource group rg-zstream-upgrade-4-21-rj5gpm, error: failed waiting for cluster=\"cluster-zstream-4-21-49s7bd\" in resourcegroup=\"rg-zstream-upgrade-4-21-rj5gpm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-49s7bd in resource group rg-zstream-upgrade-4-21-rj5gpm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-r4s4xw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: failed waiting for cluster=\"cluster-nightly-5-0-r4s4xw\" in resourcegroup=\"rg-nightly-5-0-thm68z\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-r4s4xw in resource group rg-nightly-5-0-thm68z, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}