{"meta":{"start_date":"2026-07-20","end_date":"2026-07-26","start_at":"2026-07-20T00:00:00Z","end_at":"2026-07-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T02:55:09Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":60,"failed_runs":32,"raw_occurrences":184,"matched_occurrences":178,"runs_affected":26},"rows":[{"environment":"prod","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":125,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.666666666666667,"anchor_occurrences":125,"after_last_push_count":125,"trend_counts":[0,82,42,1,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006e58a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rp6jsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"name\": \"174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T12:40:43.885322535Z\",\n      \"endTime\": \"2026-07-22T12:59:47.104093689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006e58a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rp6jsc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"name\": \"174515fd-ae84-46d9-b491-3d03c175e06c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T12:40:43.885322535Z\",\n      \"endTime\": \"2026-07-22T12:59:47.104093689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079895694151258112","occurred_at":"2026-07-22T11:45:50Z","signature_id":"8230da4579d8f1c9a9e785fcf317a05c5f292d888589ee13bcdc71ffbd9d3f02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"0012df4f80a07c3b6a174e45ee4b577b91c683510b3ea91652ed742beb80e3a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"007cf65197ae2bd9e8f2f7404c2533a5b9e03fc988753ffd47ce5e0c72d520e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"0da9a6549c113196b6efc7951a0a18523289a97e22ff686f54dfd7e467f095b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"2bcc421bad380fae66a24c3da5ba41037683155ae4e89d1bf10ed0a5ce05229f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"36eff7f126e8afb24c07e1a4dc4d242b4f6a538fd9427595143b9b10c3371cd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3945951f7e05773ea2b561ea7f6bdeed78c60d3a5f4fe590aa448bbbd2e9515c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3e96a226b21e30dc74c54fd3b273577eb22b5f774239fdc15a8148abede3f046","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3ea767832316ee9b0fc7ee7b62eac613c76d48aa08765715d31b65da9ceba713","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"3ef7d2dff7948260c72e4925bb6ed3d13b88e6cf1d66dfb25a785191e349e1dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"43e89e3345bb7d556171b9d391dc2538b1e10d9256a72fed3572449c69e0913f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"441d8fa32423755b824b03f0ed9756d2f6f572771ded2eeacd532359e6529d9c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"4443d87be4b037676b273fcbae9fe676e5dc3fc90bee3f49b560fe421976d2ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"5e36b091c1c34d3b51e47b9b6dd43f58cd9824f8e9079daacb163895f2200c4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"62e57ebc3b6e09155e29b65586f7eaa5453e4f355488112ed6ef3084f651edb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6476a8257036dba2ef91b5f15019cbdd5701e7b188929c2fee04923ee80488ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6526809d46dcfa990c8305b84fe9819b03f13654c0fe7e2ce4e680af90a63107","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6ce216fca325d3fb38de5d65149fc99e7777223055b036d53766b1a5ebc53e24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6d8a9aa9e4af877217edf30f3c6c25e1a8dd51ae566327ed7d560207914b8d6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"6ed96bf3c4348483e2e6c1462330c35308c01990058d8eabde900f6ec36cdfed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"79cb067dd7fce633fcf9600b882f8e03becb10a7fef0f4052b0f878f87663aa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"7f31b126a97f6a9ae3ed0e80c1fe187e4aa077e38f647f414370c69a5be23c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"821c3738d4a7273a8d57242e8b33fc7bfe75d51cb30bd8fecd66aa86c3d0955f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"832df04ff0266c019a92ac741e0994e1c347d1dcd016e6114a9d7a19a502c011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"852f63db6e99b0fc70dc00b3651ff3a8d766a89f489846e52cb8b55f8af962d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"90c1e458b7eafcb6f78d4c32d64f8d2e321ce0392834ddf6cf523aef2723da95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"982a3df29aed4e7b407b3f82c328895ce2e66a9c7782e98b621b68dd60e800d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"9e265d39672bf2bb80b1948ac192f4778ccafb893e435adc5941967180736187","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"a207a9f75a81edc168e301bf11e3bc2b58753e11ec0d8aa2c76c09742fb52bb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"a45a7c891b05805a6f24cd2392de01dd72e32ba8f0885fbf071bdb3f29969caf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"bc9487be47906f8adc0829bfaeb7891f9516a919d5c701126df8d3c1a2732e0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"c51c4d2e9aa454582bfa293dcddf09e5e62340b8d5506cbcd60056b871eff4fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"cbd57e43d8472457315665ef7b2a188b7fcdec695f9e63105ea81d3a101c61c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"cd2b4b061f6b7f5f091d02b70ed0abb0daa4b65b9a703f77ede69a77a1f5ae57","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"d87b62a2d298d16c9744cb8fdfcc55133bcde0f9346809cb0c3f963f838fdce4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e415c01a07919d6141e32036bedd7d2780b319450f7bd4c310fde9e4dd4fdfea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e48b650eeda4387d00133fbd7828d69f6b4d3d2563d1d298b5bd6c5d83854933","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"e5aae627e5423ff111b42a61998101a8f96b6b5c5b9d56e5f2a8ea110d53aa16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ea5c3bad22c7b0d89ec624594917a1c4cecef86c831081ab9fa716248d9a80d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ec53c2530d341327be6cba04ecfabef2505d44dcb0ec7138064837718114594c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"ef147d5fb1533c538e021dfaed880dd33e34dc663172abf3e4ea564a4b05976c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"f13cea685ea27d03325fa3981a4a87503b664077cee826714d71eb7f40b664eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"f38b7a43d956e10a30e734485ca2f145486fc2357c54ed13974c113dfd5683b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"03ccea68e6a8b694dcafc46c74a57eef305bc4ba4bf6c778ce56d2fc61dd57d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"125e1eb21cff6d94a185f310953f766c546d0091298342f5e0d3b2b8f11d9ca3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"14f34584497f527d1fd254e155a719f6a47e264ec4152faf4578d0921b44b781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"1711f5728df2e5c297c08418880e01c60aea1816ee258b02b00c95efad4f0a29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"265b6a0676836af4e807aef8fed5b704c3947e826ca78ceeeb8620193b2499ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"288cddf07006d2c3c590457909901d860be8b9ebdf2fbf927ec2e0d37decf14c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"34aa37716134afd6f7524c995d93155893fa8044ea2c34b3c11a938895a4451c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"3702e372a06a775b7ed3e266ebc6ae35c02f7531681a330d32c38e066d0dbf75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"37ae3271f611d5ae0c32caff231fa1238016e661b29388af92b05d3e07cb3336","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"3f1f342ff7f5755b4c967ba4e5516a63554ba4129d837f57feabc2e71f247ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4194b8de2fe09b669dfd595668949649e6d0742b203f5663747d477a79d6c348","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4d4a0c495fde836e9b2267ae6dc30c2ced3ec30a939332cd344d4f3903963950","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4ec9b238739f8013c9d38e8bb0fd3a4a49a36f255a2ee65d8f03d1caa2b000a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"5311ecd3697fc9f18c939f8dd5620e37778a9eb72c6a9cd3f55aa32d8cb91a66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"554d1257a5d97b619b43073ef9e77730c3695a30dc7ed68381c9e299427ea0ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"64925d571062ab351e4970f8614937952c489f1e50feda8e46999ee55fc6366e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"65e1e9a0baaa34ed8a9245e38e5e42f14d1ec4eaf26d25799fe41e9b2ddef4b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"6ac7afa86bc2d5e90600e0fa431ab0bf71a8ef325a6e37103c2002fc89a835b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"714ae85975cb35cbc648aaf48a47b60e3e3a910cae8cd699eabc8e961d796f0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"73b3a4f6df42183726a6b451d6efe754a40a2b27033d5777a7d97226fb134c0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"7a0134fabd9660f00a24a029b6933db1038908cb4142d60247d52db5de6de2f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"8049eb3eb04c5531e21091257e373deb4dd2c7db96f486e3da914e577f9aec88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"81016f7db0c3e3dd63e6633c306735ea684491fabd3cca8d82769f62a3ddf33f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"82952c372d25be52bdb5f896a7d0c7fd134f1f78b3671581993ea57d9111643c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"85d723b8b2acab4cc8c7772a3615d97b7b9eb49c6db84efcec9a2f051d1a8f91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"889ca6784d0531d5479e0c6017f00dd044e23f2c023d0791673f40a0192a6ff1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"89557bccb2b4df5db7d906bb711a4915dfe553406ec56e8185e7174548413e24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"91676d4a01ae25233828ea04f96c2ac2902ecb8011e42564e12278ad78aa1aed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"9eb4a664b9a274d1afc487ab37651ae198bf31417272ecc1061bd959bb65e625","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"a47a9bdda6418eeb4e80459a24d7f7ea8445dbc0f1d211229bf30769944a7c63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"ad5b44c12212f5532334b5a7018294db4764bc67345d1e6f4be50920237aac9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"add34bd17b15b0e5f3baa3660324a51f7fc57c1aa4c40a95f7ba5f5d5a3af52d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"b10be977c5fd6622b4608d477b0a9cd716d109a55379e9dca73b913ef5093aa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"bd236ac6ec5794320c2066b92f8aea4dc8c6219e3dcf658f4a63ea1b65873bce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d44f04715e3fc9b42e2b2b692914e2eae01d079b82aef9e4defbe36104eee6f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d81a152b208949a1d6b2e1fe144bda9593c322e64c29e592658f13d1f358e86e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d9be2042771d4fcf101626ec2a5d59b4d10f0b7f28aee5d84055a7ea7d86373b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"db7264e513dadaf676a31f0d7b1ab194ebcc3bd6cb073e2000a43d29e1527cb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"db9139d39257b8cc2ffd82462d8fde2cffc5d888dca00eefbb3a3f6ce61a43e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"e1c181b0ec5dd4f32f696abf57f3897e3d3ccf2593d37f74a0000ca1f79f2764","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"e6fb9d50a85afd951185ada7264e12cbc062b6c2d0863a724ca8b0bdb5d59230","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"0034558eead0b80d52546e7575197511515ec3d6fa978947e41fe56d1b8dc0ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"0829537e94cd357a37523e33879e0e7e22af64f406be9b39e166b0f3fa1db608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"1ba3f6f8325991f02948a4154356510c1249ac6272a12bae716bd95bed6e76a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"1eb11187eefe0b01a1fa6cc9a8e1ec084c4abe167cbe4877ca1d9800d8c201ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"275581ffbc1cf8d0c51e95c081ed302acf95313ca60b12f03dbeebfeed6f2c5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3a3e1ebdf2af97f66ec9ef78ecb1afe1fb84673af192d4e99b9ebac1bd53db84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3cee5acabea09afd20fb8c308fc2e28b46dce646f25113002c6b57a3edc86e4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3e980bc9c2c2563cced522728967775c7c3b9bd16bf2013aa99e098a1290dab4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"403b05be2c136111f81b99d81a31731e246e1d1701de8e6d380aceec11313d30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"415f5b5973ba3f3ee85b298687a47fadedce7293bdc49d77aa6a71e464b88fc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"423e79c0bd7d14affff972c4fb9cbe5fe4860e6a955ae7ba8d21ec78bceb77fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"435316384ebf44ef449ed059b08798d4179c73e4fec5c50dee33eaede524a944","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"450380f4d7d6b972a7ee9e69bef0befd4fb47db16ae4787f70b470f4ead457f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"48e8d90cc93d877691af172e6ec9ce1cbf409e9c0749017c9ad86aa87c4eea0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4adf1d1d4510d6acde5ea4f682631337eef5d8ce5834433a0b9b23e4af0dd41a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4e01e47c19ed58d4cb8d2a8f24b3122e880837292a508157c286937fd0f35b84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4ea27abf7b6d110a622804fbc493b5e0ccf63ffca1e5c4741ed58f14ea7b9019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"51686efd47937541b795284eabc7b92c47a94bf2f18851f1522b758fa9d3de37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"58c983d115a51045c4dc82de9235833a1e9b75ec570b84d7e4598afe936ac62e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"63e65a37b37f6351eb6b8822e95a229ef45f8da6b16599a8cc09a5bf14c22b6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"642426adbe17e096821e1059b47daeef68c4b1aff59c8cf307463e5911048a68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"71677fd8d99a94c4e313dfaba10dd7a548ca9df1c53cef2d7ecdf60cc2bbcbe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"8a8d6f05b6dcccf74f063e9e884d589a943e2450c74ae8f06a0e16c3e05d42bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9707201f4c3543174d8c2063e2963ddfc92c45cddea94c8422c38fda847d1c86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9cdf4b0d450d675e7353c83c727667ff2ca957686fd721c48d61cbeba18b0401","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9f1a9e027c196576a3560509efe6090b4073d1d3f8fe6efe5c81506bd99aa7cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"a8e03a7b26fd123e4dd6455e921075b3852c4e1e7487e4d00ead13c966b1f8bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"ac3d836b0cc84b6d8e6cbd78dcc6f905c310e0ae1a869903ab9785d6f9f7c79d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"afb41fc4fca05aecf9be71e40e15240a89a9e0ddb93a288c8fe5c13030ff05e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"b889d85b96f7da35ce7694e811281973ebed2db00ab6b5be37bf7f7026eea10a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c1d298712c5c3ea2c0294d603cda00c713c367daeedb531b3b9828faf8e5582b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c2b3e56093fa347ac44a6564dc83b18e1731c958bd905686c717ec056b725ce7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c3c54235f5bab1ed193af1a8bf1c53c0ebaaa00d911fbec180e364f4a51f51ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c9ef03a3c161cff519a0cc77dd1fa8a976c7e422f3986b110e834eef903df89a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"d1b2d1543f80338156eab672e9aabc48a634a2e3aed6e7362f6aeae1408ad8a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"e11b11903d7834d0633cec99c8e016a5437f7fb4b223d336b68e93d4889ded6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"e63bfecd2e80cd34d843b393871c75a9db632999606092b6f2b7879d05c4a6ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"ef690185b55d76436da7c8d08b241833303008c4ac0da413e4ac3bbff2970319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"efdc91261b3dc5be81a96c6a9bdf759132711c9bbb25b59724e796987c7c1da9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"f1229bb403839cad4628996a6410a9af531a5da6f8e4d9788b841ca3b4a79600","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"fe57b1f7e029db16afe6d13d5ca1bd16e0a7f7981e23844e85e0d5c49e7dc6e7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.666666666666667,"anchor_occurrences":11,"after_last_push_count":11,"trend_counts":[0,0,8,2,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e0aa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9n27h62csfxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"name\": \"032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T23:04:31.918481763Z\",\n      \"endTime\": \"2026-07-23T23:23:43.694668207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e0aa0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9n27h62csfxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"name\": \"032e5270-928f-475d-9dae-95082a8a0af4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T23:04:31.918481763Z\",\n      \"endTime\": \"2026-07-23T23:23:43.694668207Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005871c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-sj5lp4s8hdhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"name\": \"913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:04:09.655224875Z\",\n      \"endTime\": \"2026-07-22T02:23:21.611762653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005871c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-sj5lp4s8hdhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"name\": \"913f1b53-79a1-4c56-90f3-f94f835b8c75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:04:09.655224875Z\",\n      \"endTime\": \"2026-07-22T02:23:21.611762653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-qnfrfst59rxk/cluster-candidate-4-20-z9f7qm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-z9f7qm: failed waiting for cluster=\"cluster-candidate-4-20-z9f7qm\" in resourcegroup=\"rg-candidate-4-20-qnfrfst59rxk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"name\": \"952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:05:40.15224731Z\",\n      \"endTime\": \"2026-07-22T02:24:41.285079074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-qnfrfst59rxk/cluster-candidate-4-20-z9f7qm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b10b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-z9f7qm: failed waiting for cluster=\"cluster-candidate-4-20-z9f7qm\" in resourcegroup=\"rg-candidate-4-20-qnfrfst59rxk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"name\": \"952533d9-7ad3-4b91-b8b8-23d93e01328b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T02:05:40.15224731Z\",\n      \"endTime\": \"2026-07-22T02:24:41.285079074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2080427936950784000","occurred_at":"2026-07-23T23:00:47Z","signature_id":"703d20a7268c58a9e79f73180f6f920ace778269f7cc2f8e56f674ec9aaace7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"575d927936ae879900e3bbf127a88fd09b1860584bee57c70dfe366e81ab46e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"f25a33fd6dcb03a98d82676f51c425eade08f236835c100a86a21ec4ac380b42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","signature_id":"0755e5951a373ee7739f2da37242ca447fa6e529080b3ff77c5a94299cead45f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","signature_id":"e1a1c883dcefa03cf51fb3c9f2516413ee3f4efc3a25f372165982215efe1c15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"0b60a1c8582172801768ad634340a733b420bceb7a417943da0687ed9f68b890","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"41a62befca0336e9168b3f44954bf812cebf78b2ccbce1b2a930c0c01f99c53f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"7212f4228fcb02a7c61725079499e0e2db3beea340f52127f257bbd6a5a4fde9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"98902a321d14a054647d3e70ac37d774f906fdf4e97287d1c3c47c88206d1e19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"bf8595f1f1b6f60837a216d52bfebf5779dc79f95e44ffe1574537f650c5689a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"cf71ca211aad8a02c62dc8a01cf969bfd91e010e4142091f8852b71ff5136236","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":15,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,1,2,4,2,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080267996604928000","occurred_at":"2026-07-23T12:25:14Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2080110844716257280","occurred_at":"2026-07-23T02:00:46Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080037924878946304","occurred_at":"2026-07-22T21:11:01Z","signature_id":"a8557b42abcd154be2dbc1d4e7697becab9013bfd159699fe454c3dbe99a6090","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079895431134842880","occurred_at":"2026-07-22T11:44:47Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079748452178202624","occurred_at":"2026-07-22T02:00:45Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079731450374000640","occurred_at":"2026-07-22T00:53:11Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":7,"failed_runs":7,"run_impact_percent":11.666666666666666,"anchor_occurrences":8,"after_last_push_count":9,"trend_counts":[1,1,0,3,4,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062200\u003e: \n    failed to create NodePool np-hdmwqd: failed waiting for nodepool=\"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\" in resourcegroup=\"rg-candidate-4-20-grr9djf6qtvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"name\": \"c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T08:27:34.915224416Z\",\n      \"endTime\": \"2026-07-23T08:46:40.245736142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062200\u003e: \n    failed to create NodePool np-hdmwqd: failed waiting for nodepool=\"np-hdmwqd\" for cluster \"cluster-candidate-4-20-hdmwqd\" in resourcegroup=\"rg-candidate-4-20-grr9djf6qtvt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"name\": \"c91f066e-3821-44ee-bece-edbab2bb31db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T08:27:34.915224416Z\",\n      \"endTime\": \"2026-07-23T08:46:40.245736142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00283c700\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5676c2nc84jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"name\": \"81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T07:46:53.376501029Z\",\n      \"endTime\": \"2026-07-23T08:05:58.659570015Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00283c700\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5676c2nc84jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"name\": \"81d6e351-7e5e-49b4-85a5-302b276cc8f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T07:46:53.376501029Z\",\n      \"endTime\": \"2026-07-23T08:05:58.659570015Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029b0ac0\u003e: \n    failed to create NodePool np-hl8829: failed waiting for nodepool=\"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\" in resourcegroup=\"rg-candidate-4-21-tv28gtjd9f9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"name\": \"d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T04:16:36.200454364Z\",\n      \"endTime\": \"2026-07-23T04:35:50.806314695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0029b0ac0\u003e: \n    failed to create NodePool np-hl8829: failed waiting for nodepool=\"np-hl8829\" for cluster \"cluster-candidate-4-21-hl8829\" in resourcegroup=\"rg-candidate-4-21-tv28gtjd9f9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"name\": \"d4916d8a-cfac-4b24-ba00-5346b085835c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T04:16:36.200454364Z\",\n      \"endTime\": \"2026-07-23T04:35:50.806314695Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080188520843972608","occurred_at":"2026-07-23T07:09:26Z","signature_id":"0984e8df881596c9b834e29d1fcae2818b2d904758affc92d244ae142cb90020","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080188520843972608","occurred_at":"2026-07-23T07:09:26Z","signature_id":"463ea67d556eeb787e73f24b0e88b683d070445d3805c355c27706ee59576f5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080132612415819776","occurred_at":"2026-07-23T03:27:16Z","signature_id":"55243e92d198af4223d7533882b14acb2a62c7ed804ca25f81a7e1527b9f37e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2080110844716257280","occurred_at":"2026-07-23T02:00:46Z","signature_id":"61dff9a614aba5d6b259b363f11b1fe06cb1f30dcb1be30b825480e34108aa17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080036190387441664","occurred_at":"2026-07-22T21:04:07Z","signature_id":"78a8d6258739afb8e12558c28fcccc4e14633dba9764e96e9ace0c147dbc3c51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079792930498809856","occurred_at":"2026-07-22T04:57:29Z","signature_id":"9398019cf66f84e56213a10b2bdac60b520839a24df306fb10defedff1a34ec1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079730275666890752","occurred_at":"2026-07-22T00:48:31Z","signature_id":"296fba6b5630a0876c842ef157378bd0a4ac3e8b1a977f7b931413e0fc9f61f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","signature_id":"c15750545600e820878efaa487132ca680dc610368be4906d55af70ae65160b6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"clusternp128-nzwfdf99d2bd\", caused by: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":6,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.3333333333333335,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,1,0,0,0,0,5,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-06","2026-07-13"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-19T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ab02a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"private-keyvault-hzd8f756pkjx\", caused by: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hzd8f756pkjx\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-hzd8f756pkjx, error: context deadline exceeded, error: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hzd8f756pkjx\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-hzd8f756pkjx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ab02a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"private-keyvault-hzd8f756pkjx\", caused by: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hzd8f756pkjx\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-hzd8f756pkjx, error: context deadline exceeded, error: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hzd8f756pkjx\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-hzd8f756pkjx, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00172e378\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\", caused by: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hszcm2jz5pbv, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hszcm2jz5pbv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00172e378\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\", caused by: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hszcm2jz5pbv, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hszcm2jz5pbv\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-hszcm2jz5pbv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fc0090\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"clusternp128-nzwfdf99d2bd\", caused by: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fc0090\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"clusternp128-nzwfdf99d2bd\", caused by: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nzwfdf99d2bd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-nzwfdf99d2bd, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"1e56dab0ef8016bc4be8e9c59d10fea1b0be2bd7f744555554b61d3d8add2ab3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"2497c00f2c6f0b3815cd3a846155256e90e6796c706b2489c85c4e9a86f79e6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"498efaa1c876db80848bf71276aa615289766c17dc20189c8e7ba7b668f09742","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"9cbffc633e245a65e53060d0b4521a02a382b8b9f99325a2172b6249a4d57ab9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"d44ea016b25aa9fc93b40395c53b4a5ff01abf0f9dbd06d8e9b92cba66ec7fed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","signature_id":"0d4e6c626552d37165a629be9c672bc072d0e8affb1fa7528cee84809b6aefcf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.666666666666667,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,3,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079074941612855296","occurred_at":"2026-07-20T05:24:28Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":6.666666666666667,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,3,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":43,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","signature_id":"db6c0aad1a7eb3447967f2ef43e12a6e5c1c8c6982e603975af0a727456df14d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"a41a9dd0285ca9f12cb560711358249e2025fbe7cf12cf82a42396190c9042c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079340761836425216","occurred_at":"2026-07-20T23:00:44Z","signature_id":"4127a9187d48928f29732c57d0737caa49f7885f5a800e49e26c0ea8b42fe648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"56f87fb5a807646e254bd4c95e1d7baae5a9e2cc8f9b0f9e1c6f0454d4a97551","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":3.3333333333333335,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,1,0,1,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-16T09:01:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-dpbrz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbe3e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-zszzt.apps.aro.x0o8h6x6l7k9u1f.e0w5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-nightly-4-23-dpbrz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbe3e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-zszzt.apps.aro.x0o8h6x6l7k9u1f.e0w5.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a0b110\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a0b110\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2080065545201258496","occurred_at":"2026-07-22T23:00:46Z","signature_id":"4a18545286c04da4906905b26b99ad474db07bd58ff38b1f675e1e3535e445a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079317230851461120","occurred_at":"2026-07-20T21:27:14Z","signature_id":"ed61138e00b6a5005e4f6c12099e2f0f17e5485a149e9e9870bbab9d9d102348","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-08T14:15:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffc3cc57520\nrsp    0x7ffc3cc574d8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7\nr13    0x1\nr14    0xc957640\nr15    0x7\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffc3cc57520\nrsp    0x7ffc3cc574d8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7\nr13    0x1\nr14    0xc957640\nr15    0x7\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffe78eb87a8\nrsp    0x7ffe78eb8760\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x4c\nr13    0x1\nr14    0xc957640\nr15    0x1\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffe78eb87a8\nrsp    0x7ffe78eb8760\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x4c\nr13    0x1\nr14    0xc957640\nr15    0x1\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9dfae6e5978c1295fffad1fa285c0f56737b3e5ba630e35f78fc7c230070cc36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"b6f4e9f1f6c7f0e0d12f6274441cf72f39009053ff85d2d81ef74cebb5deb612","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-jlqwkzs7jqhj, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-22"],"prior_runs_affected":2,"prior_last_seen_at":"2026-06-23T23:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-jlqwkzs7jqhj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001296270\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-jlqwkzs7jqhj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-jlqwkzs7jqhj\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001296270\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-jlqwkzs7jqhj, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"ec33c7603488f2dc76c9b19b94d20967736d2be7d2e9e2ef61b70a03c6f4f49a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2a5131f222749c0c944dc75ca14b7d24e45ce57c0da62c7a9019b2ce22a3e31d","failure_pattern":"ERROR CODE: InternalServerError; detail code DenyAssignmentAuthorizationFailed; detail message Access denied by deny assignment for action \"Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action\".; provider Microsoft.Authorization","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00003c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xxxldl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"name\": \"9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T08:31:33.476055825Z\",\n      \"endTime\": \"2026-07-22T08:50:37.393817181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00003c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xxxldl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"name\": \"9c4f6c2a-fe7f-4a6d-b9a4-e7fb6b5215d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-22T08:31:33.476055825Z\",\n      \"endTime\": \"2026-07-22T08:50:37.393817181Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-nxzhn\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-nxzhn'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n      - VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-8skrs-wwbw4\": POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: DenyAssignmentAuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DenyAssignmentAuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' has permission to perform action 'Microsoft.Compute/virtualMachines/retrieveBootDiagnosticsData/action' on scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8skrs-wwbw4'; however, the access is denied because of the deny assignment with name '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed/providers/Microsoft.Authorization/denyAssignments/4e1d5354-563b-53f0-a663-9624ce82fcc6' and Id '4e1d5354563b53f0a6639624ce82fcc6' at scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xxxldl--managed'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079841571435450368","occurred_at":"2026-07-22T08:10:44Z","signature_id":"ea30ba3347f94c61ae7d6827ed5775c854aeecaddab6076a77ef7240e95ba09b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-bvp8bvjdldql/cluster-nightly-4-23-6252tm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b819a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-6252tm: failed waiting for cluster=\"cluster-nightly-4-23-6252tm\" in resourcegroup=\"rg-nightly-4-23-bvp8bvjdldql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a07651a-a9b8-414c-8974-3522743736b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a07651a-a9b8-414c-8974-3522743736b5\",\n      \"name\": \"7a07651a-a9b8-414c-8974-3522743736b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-25T23:30:22.26941828Z\",\n      \"endTime\": \"2026-07-25T23:49:39.833761299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-bvp8bvjdldql/cluster-nightly-4-23-6252tm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b819a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-6252tm: failed waiting for cluster=\"cluster-nightly-4-23-6252tm\" in resourcegroup=\"rg-nightly-4-23-bvp8bvjdldql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a07651a-a9b8-414c-8974-3522743736b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a07651a-a9b8-414c-8974-3522743736b5\",\n      \"name\": \"7a07651a-a9b8-414c-8974-3522743736b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-25T23:30:22.26941828Z\",\n      \"endTime\": \"2026-07-25T23:49:39.833761299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2081152736203640832","occurred_at":"2026-07-25T23:00:52Z","signature_id":"eaf4ffea5ec685d7eb4149d21d9d11f620f8f04dae8a30c961dea770a5bad65e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"69b586e970c0cce85494b13fd30b9055a1bcead75f883e0e702343732b8f47e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c59269c250bd09f866d89f5097669316613ce7d5e4a7268ed28a2ccc21e5134c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","signature_id":"8762cfe09fbb635fdeb6d2aeb186ff27d5dcfb284fb7d8cd73b9f9066590a09b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b43e7c6c150476d7400be2924e381be5a30b739ff1a4de2439b70edbc0e574e4","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Network","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"075407059f3a5b3d581b963f2098a2be85da109dc5ca5fcc1562bfbb79dd101f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b8223127140c833357088cdafd7290dc7db01297e09b7695d10e69e7ef3961f2","failure_pattern":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.6666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a57d0\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a57d0\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2080087615616323584","occurred_at":"2026-07-23T00:28:28Z","signature_id":"1c30b2761273ca4099159b93719edbce4a961fd02fb2ee58f934ccaddd0614c0","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}