{"meta":{"start_date":"2026-04-22","end_date":"2026-04-22","start_at":"2026-04-22T00:00:00Z","end_at":"2026-04-23T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T17:16:24Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":11,"matched_occurrences":11,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-04-16..2026-04-22 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e508b8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fjrggl\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c6a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9vtz75\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c6a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9vtz75\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00196a408\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hk2d79\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc00196a408\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hk2d79\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"48159465318ae7cb16fd7133a8f7cdcf03097f41d54cb30cea70261ed9054c8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"b03d1fadc884dc16e19a954a64e384b01f7091cef8db05565dd9a7755a124e12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","signature_id":"7f1df1cc9a46d12be1958e532b4c5752baeb878aacfd79d0bcd706518d43d67e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"f80b7ea1525f797da1e97c41f8eb510f1337f4e1acde91deda06ef14f08873dc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-04-16..2026-04-22 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-23","2026-04-06","2026-04-13"],"prior_runs_affected":7,"prior_last_seen_at":"2026-04-15T09:25:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019264e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019264e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.z8b5g9a8l3e9r0o.idqn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.z8b5g9a8l3e9r0o.bcp0.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc0007cd3c8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc0007cd3c8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.i1s3o9k1e1z3u0n.4xyn.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.i1s3o9k1e1z3u0n.4nkk.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a46d38\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:141]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a46d38\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.s5s7v5v8m6t7f6o.wmw2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.s5s7v5v8m6t7f6o.mp5p.uksouth.aroapp-hcp.io\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"6736c55513db9c62b4e7b1d4817a91d50f048565ffe1d4242804bd6d2d498e88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047024789423394816","occurred_at":"2026-04-22T18:48:36Z","signature_id":"7fe72b9b19f5e7fead9f3de4a2cd748b499d2f808a00b615669fe1eafe479696","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"17fa8a755d5fbe5fb9413c3d9a98023fd7c2539d192561972224f6cbee3c6d7c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-04-16..2026-04-22 UTC","prior_windows_present":4,"prior_window_starts":["2026-03-23","2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-18T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000910630\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-kwwz97, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000910630\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-kwwz97, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kwwz97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-t2vg4j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-kwwz97 in resource group rg-cp-ystream-upgrade-4-21-t2vg4j, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010f82d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qf6tsk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: failed waiting for cluster=\"np-version-upgrade-cluster-qf6tsk\" in resourcegroup=\"rg-np-version-upgrade-qf6tsk-8dlq72\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010f82d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qf6tsk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: failed waiting for cluster=\"np-version-upgrade-cluster-qf6tsk\" in resourcegroup=\"rg-np-version-upgrade-qf6tsk-8dlq72\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qf6tsk in resource group rg-np-version-upgrade-qf6tsk-8dlq72, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"a020305e40755460769f708621143631e144b7929851ebafdb405f835006c286","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046914038591393792","occurred_at":"2026-04-22T11:28:31Z","signature_id":"333fe7ca0e7f018212d5389f30014334b1c60cbf991ac7284951a7c4115a1a44","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"66bc8d9427d72c6b659d24636018e42338a779f44749a6537f189769d48e7cd4","failure_pattern":"ERROR CODE: ConflictingConcurrentWriteNotAllowed; detail message The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ConflictingConcurrentWriteNotAllowed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-16..2026-04-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171e978\u003e: \n    failed to cleanup resource group: ordered cleanup workflow failed for disabled-image-registry-l5b2fs: Delete non-networking resources: failed deleting disabled-image-registry-hcp-cluster (Microsoft.RedHatOpenShift/HCPOpenShiftClusters): DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-l5b2fs/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ConflictingConcurrentWriteNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ConflictingConcurrentWriteNotAllowed\",\n        \"message\": \"The operation was interrupted by a conflicting concurrent write on the same entity. Please retry later.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046956383668015104","occurred_at":"2026-04-22T14:16:47Z","signature_id":"04e67810388081d177ee7c2235f817b3562a0317dade3675b8b04ad92e0e8fa0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":38,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-04-16..2026-04-22 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-30","2026-04-06","2026-04-13"],"prior_runs_affected":18,"prior_last_seen_at":"2026-04-16T02:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0001bfaa0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xv5wb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"name\": \"35394bfc-4a06-43b3-aa53-a1d60828c361\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-22T09:47:15.284582065Z\",\n      \"endTime\": \"2026-04-22T09:48:23.845505796Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046879318772027392","occurred_at":"2026-04-22T09:10:33Z","signature_id":"200e82b61130b0e496380db62fba3d48341980e2169169844b87db2b8a56e38f","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}