{"meta":{"date":"2026-04-22","timezone":"UTC","generated_at":"2026-08-30T06:44:20Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-22T18:48:36Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"c27cb68296a61df5824b16e55885e380a606d12f8f7931dc6463b822fb4676df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","failed_at":"e2e","signature_id":"6736c55513db9c62b4e7b1d4817a91d50f048565ffe1d4242804bd6d2d498e88","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019264e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019264e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io"}},{"row_id":"e75237e3aafeaf9f6028640e9e8e90cbdcff3a7a3a23ce3d099991b3cea9c9db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","failed_at":"e2e","signature_id":"7fe72b9b19f5e7fead9f3de4a2cd748b499d2f808a00b615669fe1eafe479696","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc0007cd3c8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc0007cd3c8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io"}},{"row_id":"fbb73d3c546b63b12b2a4cbdfa77bb1885a094bacb7c8838d640ee69ebc63ebf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","failed_at":"e2e","signature_id":"48159465318ae7cb16fd7133a8f7cdcf03097f41d54cb30cea70261ed9054c8f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-22T14:16:47Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3e3c0ed104f494ebb89ae2f91ee0f530c0f7a594f946d7d0010b76b6abd3daf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","failed_at":"e2e","signature_id":"04e67810388081d177ee7c2235f817b3562a0317dade3675b8b04ad92e0e8fa0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"66bc8d9427d72c6b659d24636018e42338a779f44749a6537f189769d48e7cd4","failure_pattern":"ERROR CODE: ConflictingConcurrentWriteNotAllowed; detail message The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ConflictingConcurrentWriteNotAllowed"}},{"row_id":"47464d55405bf3c8f50f9dfaea7310ac10972d48ba9d63cb6cdaceccdf010622","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","failed_at":"e2e","signature_id":"b03d1fadc884dc16e19a954a64e384b01f7091cef8db05565dd9a7755a124e12","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c6a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9vtz75\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c6a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9vtz75\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"4c98c3437fc47260f297c9a27efefcbadb46be18a9743b4f9db3ba1044efdd8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","failed_at":"e2e","signature_id":"a020305e40755460769f708621143631e144b7929851ebafdb405f835006c286","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000910630\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-kwwz97, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000910630\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-kwwz97, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","66bc8d9427d72c6b659d24636018e42338a779f44749a6537f189769d48e7cd4","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-22T11:28:31Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0a8ca9f949554c293b88d4ef6a9f13ffca6c7d293244ea2111cb4a0fc760f1f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","failed_at":"e2e","signature_id":"333fe7ca0e7f018212d5389f30014334b1c60cbf991ac7284951a7c4115a1a44","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010f82d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qf6tsk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: failed waiting for cluster=\"np-version-upgrade-cluster-qf6tsk\" in resourcegroup=\"rg-np-version-upgrade-qf6tsk-8dlq72\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010f82d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qf6tsk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: failed waiting for cluster=\"np-version-upgrade-cluster-qf6tsk\" in resourcegroup=\"rg-np-version-upgrade-qf6tsk-8dlq72\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded"}},{"row_id":"a52f802a9141e1c287faf3c5a55e814a9063cefefe6a8b4b347ef83c64a2e2ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","failed_at":"e2e","signature_id":"7f1df1cc9a46d12be1958e532b4c5752baeb878aacfd79d0bcd706518d43d67e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00196a408\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hk2d79\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00196a408\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hk2d79\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-22T09:10:33Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0dd406f7de1b0cf3d97b2e80297350118437098dd3374d58e6468afa9b356cc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","failed_at":"e2e","signature_id":"200e82b61130b0e496380db62fba3d48341980e2169169844b87db2b8a56e38f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"600104f1d206effe0f33361b609ad9665b05db75c65dd0cf30f9efc55e2fc957","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","failed_at":"e2e","signature_id":"17fa8a755d5fbe5fb9413c3d9a98023fd7c2539d192561972224f6cbee3c6d7c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a46d38\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a46d38\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io"}},{"row_id":"a7fb11f40ca3e87b5336c0030d6cad9a6f000995ddf1bb70d51eeb3aef0deef0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","failed_at":"e2e","signature_id":"f80b7ea1525f797da1e97c41f8eb510f1337f4e1acde91deda06ef14f08873dc","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e6a360\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bkrk7k\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e6a360\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bkrk7k\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046771096715268096","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-22T02:00:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}