{"meta":{"start_date":"2026-07-21","end_date":"2026-07-21","start_at":"2026-07-21T00:00:00Z","end_at":"2026-07-22T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T06:05:45Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":3,"raw_occurrences":56,"matched_occurrences":56,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":42,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":42,"after_last_push_count":124,"trend_counts":[0,0,0,0,0,82,42],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009721e0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hkl2lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"name\": \"6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.336144366Z\",\n      \"endTime\": \"2026-07-21T08:07:14.180527663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009721e0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hkl2lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"name\": \"6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.336144366Z\",\n      \"endTime\": \"2026-07-21T08:07:14.180527663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"0012df4f80a07c3b6a174e45ee4b577b91c683510b3ea91652ed742beb80e3a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"007cf65197ae2bd9e8f2f7404c2533a5b9e03fc988753ffd47ce5e0c72d520e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"0da9a6549c113196b6efc7951a0a18523289a97e22ff686f54dfd7e467f095b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"2bcc421bad380fae66a24c3da5ba41037683155ae4e89d1bf10ed0a5ce05229f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"36eff7f126e8afb24c07e1a4dc4d242b4f6a538fd9427595143b9b10c3371cd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3945951f7e05773ea2b561ea7f6bdeed78c60d3a5f4fe590aa448bbbd2e9515c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3e96a226b21e30dc74c54fd3b273577eb22b5f774239fdc15a8148abede3f046","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3ea767832316ee9b0fc7ee7b62eac613c76d48aa08765715d31b65da9ceba713","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3ef7d2dff7948260c72e4925bb6ed3d13b88e6cf1d66dfb25a785191e349e1dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"43e89e3345bb7d556171b9d391dc2538b1e10d9256a72fed3572449c69e0913f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"441d8fa32423755b824b03f0ed9756d2f6f572771ded2eeacd532359e6529d9c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"4443d87be4b037676b273fcbae9fe676e5dc3fc90bee3f49b560fe421976d2ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"5e36b091c1c34d3b51e47b9b6dd43f58cd9824f8e9079daacb163895f2200c4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"62e57ebc3b6e09155e29b65586f7eaa5453e4f355488112ed6ef3084f651edb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6476a8257036dba2ef91b5f15019cbdd5701e7b188929c2fee04923ee80488ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6526809d46dcfa990c8305b84fe9819b03f13654c0fe7e2ce4e680af90a63107","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6ce216fca325d3fb38de5d65149fc99e7777223055b036d53766b1a5ebc53e24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6d8a9aa9e4af877217edf30f3c6c25e1a8dd51ae566327ed7d560207914b8d6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6ed96bf3c4348483e2e6c1462330c35308c01990058d8eabde900f6ec36cdfed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"79cb067dd7fce633fcf9600b882f8e03becb10a7fef0f4052b0f878f87663aa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"7f31b126a97f6a9ae3ed0e80c1fe187e4aa077e38f647f414370c69a5be23c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"821c3738d4a7273a8d57242e8b33fc7bfe75d51cb30bd8fecd66aa86c3d0955f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"832df04ff0266c019a92ac741e0994e1c347d1dcd016e6114a9d7a19a502c011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"852f63db6e99b0fc70dc00b3651ff3a8d766a89f489846e52cb8b55f8af962d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"90c1e458b7eafcb6f78d4c32d64f8d2e321ce0392834ddf6cf523aef2723da95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"982a3df29aed4e7b407b3f82c328895ce2e66a9c7782e98b621b68dd60e800d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"9e265d39672bf2bb80b1948ac192f4778ccafb893e435adc5941967180736187","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"a207a9f75a81edc168e301bf11e3bc2b58753e11ec0d8aa2c76c09742fb52bb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"a45a7c891b05805a6f24cd2392de01dd72e32ba8f0885fbf071bdb3f29969caf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"bc9487be47906f8adc0829bfaeb7891f9516a919d5c701126df8d3c1a2732e0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"c51c4d2e9aa454582bfa293dcddf09e5e62340b8d5506cbcd60056b871eff4fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"cbd57e43d8472457315665ef7b2a188b7fcdec695f9e63105ea81d3a101c61c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"cd2b4b061f6b7f5f091d02b70ed0abb0daa4b65b9a703f77ede69a77a1f5ae57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"d87b62a2d298d16c9744cb8fdfcc55133bcde0f9346809cb0c3f963f838fdce4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e415c01a07919d6141e32036bedd7d2780b319450f7bd4c310fde9e4dd4fdfea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e48b650eeda4387d00133fbd7828d69f6b4d3d2563d1d298b5bd6c5d83854933","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e5aae627e5423ff111b42a61998101a8f96b6b5c5b9d56e5f2a8ea110d53aa16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ea5c3bad22c7b0d89ec624594917a1c4cecef86c831081ab9fa716248d9a80d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ec53c2530d341327be6cba04ecfabef2505d44dcb0ec7138064837718114594c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ef147d5fb1533c538e021dfaed880dd33e34dc663172abf3e4ea564a4b05976c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"f13cea685ea27d03325fa3981a4a87503b664077cee826714d71eb7f40b664eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"f38b7a43d956e10a30e734485ca2f145486fc2357c54ed13974c113dfd5683b2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-21-r4hvsk6dxsls/cluster-nightly-4-21-r2qnp7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c7ec40\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-r2qnp7: failed waiting for cluster=\"cluster-nightly-4-21-r2qnp7\" in resourcegroup=\"rg-nightly-4-21-r4hvsk6dxsls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"name\": \"4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:27:28.08522946Z\",\n      \"endTime\": \"2026-07-21T23:46:45.381055827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-21-r4hvsk6dxsls/cluster-nightly-4-21-r2qnp7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c7ec40\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-r2qnp7: failed waiting for cluster=\"cluster-nightly-4-21-r2qnp7\" in resourcegroup=\"rg-nightly-4-21-r4hvsk6dxsls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"name\": \"4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:27:28.08522946Z\",\n      \"endTime\": \"2026-07-21T23:46:45.381055827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-20-sq54mx2hh8bz/cluster-nightly-4-20-l4wzgj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa40\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-l4wzgj: failed waiting for cluster=\"cluster-nightly-4-20-l4wzgj\" in resourcegroup=\"rg-nightly-4-20-sq54mx2hh8bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"name\": \"a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:38.885352827Z\",\n      \"endTime\": \"2026-07-21T23:24:44.270636482Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-20-sq54mx2hh8bz/cluster-nightly-4-20-l4wzgj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa40\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-l4wzgj: failed waiting for cluster=\"cluster-nightly-4-20-l4wzgj\" in resourcegroup=\"rg-nightly-4-20-sq54mx2hh8bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"name\": \"a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:38.885352827Z\",\n      \"endTime\": \"2026-07-21T23:24:44.270636482Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-jf26hw/cluster-candidate-4-21-sbclb8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed8640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-sbclb8: failed waiting for cluster=\"cluster-candidate-4-21-sbclb8\" in resourcegroup=\"rg-candidate-4-21-jf26hw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"name\": \"f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:06.106421624Z\",\n      \"endTime\": \"2026-07-21T17:42:20.33106929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-jf26hw/cluster-candidate-4-21-sbclb8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed8640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-sbclb8: failed waiting for cluster=\"cluster-candidate-4-21-sbclb8\" in resourcegroup=\"rg-candidate-4-21-jf26hw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"name\": \"f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:06.106421624Z\",\n      \"endTime\": \"2026-07-21T17:42:20.33106929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","signature_id":"0755e5951a373ee7739f2da37242ca447fa6e529080b3ff77c5a94299cead45f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","signature_id":"e1a1c883dcefa03cf51fb3c9f2516413ee3f4efc3a25f372165982215efe1c15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"0b60a1c8582172801768ad634340a733b420bceb7a417943da0687ed9f68b890","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"41a62befca0336e9168b3f44954bf812cebf78b2ccbce1b2a930c0c01f99c53f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"7212f4228fcb02a7c61725079499e0e2db3beea340f52127f257bbd6a5a4fde9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"98902a321d14a054647d3e70ac37d774f906fdf4e97287d1c3c47c88206d1e19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"bf8595f1f1b6f60837a216d52bfebf5779dc79f95e44ffe1574537f650c5689a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"cf71ca211aad8a02c62dc8a01cf969bfd91e010e4142091f8852b71ff5136236","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":22.22222222222222,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"69b586e970c0cce85494b13fd30b9055a1bcead75f883e0e702343732b8f47e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c59269c250bd09f866d89f5097669316613ce7d5e4a7268ed28a2ccc21e5134c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"8762cfe09fbb635fdeb6d2aeb186ff27d5dcfb284fb7d8cd73b9f9066590a09b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-15..2026-07-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":43,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"db6c0aad1a7eb3447967f2ef43e12a6e5c1c8c6982e603975af0a727456df14d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}