{"meta":{"start_date":"2026-07-20","end_date":"2026-07-20","start_at":"2026-07-20T00:00:00Z","end_at":"2026-07-21T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:21:50Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":14,"failed_runs":6,"raw_occurrences":95,"matched_occurrences":95,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":82,"runs_affected":2,"failed_runs":2,"run_impact_percent":14.285714285714286,"anchor_occurrences":82,"after_last_push_count":82,"trend_counts":[0,0,0,0,0,0,82],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a67ae0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4qrkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n \"name\": \"5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-21T00:13:34.291625253Z\",\n \"endTime\": \"2026-07-21T00:32:37.715507635Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a67ae0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4qrkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n \"name\": \"5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-21T00:13:34.291625253Z\",\n \"endTime\": \"2026-07-21T00:32:37.715507635Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c73aa0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wbrpm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n \"name\": \"810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-20T23:27:20.299583025Z\",\n \"endTime\": \"2026-07-20T23:46:34.265720908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c73aa0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wbrpm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n \"name\": \"810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-20T23:27:20.299583025Z\",\n \"endTime\": \"2026-07-20T23:46:34.265720908Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e086a0\u003e: \n failed to create HCP cluster np-downgrade-kfxhrl: failed waiting for cluster=\"np-downgrade-kfxhrl\" in resourcegroup=\"rg-np-downgrade-kfxhrl-2h659z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n \"name\": \"fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-21T00:15:07.341508774Z\",\n \"endTime\": \"2026-07-21T00:34:07.168232803Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e086a0\u003e: \n failed to create HCP cluster np-downgrade-kfxhrl: failed waiting for cluster=\"np-downgrade-kfxhrl\" in resourcegroup=\"rg-np-downgrade-kfxhrl-2h659z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n \"name\": \"fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-21T00:15:07.341508774Z\",\n \"endTime\": \"2026-07-21T00:34:07.168232803Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"03ccea68e6a8b694dcafc46c74a57eef305bc4ba4bf6c778ce56d2fc61dd57d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"125e1eb21cff6d94a185f310953f766c546d0091298342f5e0d3b2b8f11d9ca3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"14f34584497f527d1fd254e155a719f6a47e264ec4152faf4578d0921b44b781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"1711f5728df2e5c297c08418880e01c60aea1816ee258b02b00c95efad4f0a29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"265b6a0676836af4e807aef8fed5b704c3947e826ca78ceeeb8620193b2499ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"288cddf07006d2c3c590457909901d860be8b9ebdf2fbf927ec2e0d37decf14c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"34aa37716134afd6f7524c995d93155893fa8044ea2c34b3c11a938895a4451c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"3702e372a06a775b7ed3e266ebc6ae35c02f7531681a330d32c38e066d0dbf75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"37ae3271f611d5ae0c32caff231fa1238016e661b29388af92b05d3e07cb3336","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"3f1f342ff7f5755b4c967ba4e5516a63554ba4129d837f57feabc2e71f247ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4194b8de2fe09b669dfd595668949649e6d0742b203f5663747d477a79d6c348","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4d4a0c495fde836e9b2267ae6dc30c2ced3ec30a939332cd344d4f3903963950","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"4ec9b238739f8013c9d38e8bb0fd3a4a49a36f255a2ee65d8f03d1caa2b000a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"5311ecd3697fc9f18c939f8dd5620e37778a9eb72c6a9cd3f55aa32d8cb91a66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"554d1257a5d97b619b43073ef9e77730c3695a30dc7ed68381c9e299427ea0ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"64925d571062ab351e4970f8614937952c489f1e50feda8e46999ee55fc6366e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"65e1e9a0baaa34ed8a9245e38e5e42f14d1ec4eaf26d25799fe41e9b2ddef4b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"6ac7afa86bc2d5e90600e0fa431ab0bf71a8ef325a6e37103c2002fc89a835b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"714ae85975cb35cbc648aaf48a47b60e3e3a910cae8cd699eabc8e961d796f0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"73b3a4f6df42183726a6b451d6efe754a40a2b27033d5777a7d97226fb134c0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"7a0134fabd9660f00a24a029b6933db1038908cb4142d60247d52db5de6de2f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"8049eb3eb04c5531e21091257e373deb4dd2c7db96f486e3da914e577f9aec88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"81016f7db0c3e3dd63e6633c306735ea684491fabd3cca8d82769f62a3ddf33f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"82952c372d25be52bdb5f896a7d0c7fd134f1f78b3671581993ea57d9111643c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"85d723b8b2acab4cc8c7772a3615d97b7b9eb49c6db84efcec9a2f051d1a8f91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"889ca6784d0531d5479e0c6017f00dd044e23f2c023d0791673f40a0192a6ff1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"89557bccb2b4df5db7d906bb711a4915dfe553406ec56e8185e7174548413e24","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"91676d4a01ae25233828ea04f96c2ac2902ecb8011e42564e12278ad78aa1aed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"9eb4a664b9a274d1afc487ab37651ae198bf31417272ecc1061bd959bb65e625","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"a47a9bdda6418eeb4e80459a24d7f7ea8445dbc0f1d211229bf30769944a7c63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"ad5b44c12212f5532334b5a7018294db4764bc67345d1e6f4be50920237aac9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"add34bd17b15b0e5f3baa3660324a51f7fc57c1aa4c40a95f7ba5f5d5a3af52d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"b10be977c5fd6622b4608d477b0a9cd716d109a55379e9dca73b913ef5093aa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"bd236ac6ec5794320c2066b92f8aea4dc8c6219e3dcf658f4a63ea1b65873bce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d44f04715e3fc9b42e2b2b692914e2eae01d079b82aef9e4defbe36104eee6f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d81a152b208949a1d6b2e1fe144bda9593c322e64c29e592658f13d1f358e86e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"d9be2042771d4fcf101626ec2a5d59b4d10f0b7f28aee5d84055a7ea7d86373b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"db7264e513dadaf676a31f0d7b1ab194ebcc3bd6cb073e2000a43d29e1527cb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"db9139d39257b8cc2ffd82462d8fde2cffc5d888dca00eefbb3a3f6ce61a43e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"e1c181b0ec5dd4f32f696abf57f3897e3d3ccf2593d37f74a0000ca1f79f2764","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"e6fb9d50a85afd951185ada7264e12cbc062b6c2d0863a724ca8b0bdb5d59230","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"0034558eead0b80d52546e7575197511515ec3d6fa978947e41fe56d1b8dc0ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"0829537e94cd357a37523e33879e0e7e22af64f406be9b39e166b0f3fa1db608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"1ba3f6f8325991f02948a4154356510c1249ac6272a12bae716bd95bed6e76a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"1eb11187eefe0b01a1fa6cc9a8e1ec084c4abe167cbe4877ca1d9800d8c201ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"275581ffbc1cf8d0c51e95c081ed302acf95313ca60b12f03dbeebfeed6f2c5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3a3e1ebdf2af97f66ec9ef78ecb1afe1fb84673af192d4e99b9ebac1bd53db84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3cee5acabea09afd20fb8c308fc2e28b46dce646f25113002c6b57a3edc86e4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"3e980bc9c2c2563cced522728967775c7c3b9bd16bf2013aa99e098a1290dab4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"403b05be2c136111f81b99d81a31731e246e1d1701de8e6d380aceec11313d30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"415f5b5973ba3f3ee85b298687a47fadedce7293bdc49d77aa6a71e464b88fc7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"423e79c0bd7d14affff972c4fb9cbe5fe4860e6a955ae7ba8d21ec78bceb77fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"435316384ebf44ef449ed059b08798d4179c73e4fec5c50dee33eaede524a944","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"450380f4d7d6b972a7ee9e69bef0befd4fb47db16ae4787f70b470f4ead457f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"48e8d90cc93d877691af172e6ec9ce1cbf409e9c0749017c9ad86aa87c4eea0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4adf1d1d4510d6acde5ea4f682631337eef5d8ce5834433a0b9b23e4af0dd41a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4e01e47c19ed58d4cb8d2a8f24b3122e880837292a508157c286937fd0f35b84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"4ea27abf7b6d110a622804fbc493b5e0ccf63ffca1e5c4741ed58f14ea7b9019","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"51686efd47937541b795284eabc7b92c47a94bf2f18851f1522b758fa9d3de37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"58c983d115a51045c4dc82de9235833a1e9b75ec570b84d7e4598afe936ac62e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"63e65a37b37f6351eb6b8822e95a229ef45f8da6b16599a8cc09a5bf14c22b6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"642426adbe17e096821e1059b47daeef68c4b1aff59c8cf307463e5911048a68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"71677fd8d99a94c4e313dfaba10dd7a548ca9df1c53cef2d7ecdf60cc2bbcbe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"8a8d6f05b6dcccf74f063e9e884d589a943e2450c74ae8f06a0e16c3e05d42bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9707201f4c3543174d8c2063e2963ddfc92c45cddea94c8422c38fda847d1c86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9cdf4b0d450d675e7353c83c727667ff2ca957686fd721c48d61cbeba18b0401","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9f1a9e027c196576a3560509efe6090b4073d1d3f8fe6efe5c81506bd99aa7cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"a8e03a7b26fd123e4dd6455e921075b3852c4e1e7487e4d00ead13c966b1f8bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"ac3d836b0cc84b6d8e6cbd78dcc6f905c310e0ae1a869903ab9785d6f9f7c79d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"afb41fc4fca05aecf9be71e40e15240a89a9e0ddb93a288c8fe5c13030ff05e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"b889d85b96f7da35ce7694e811281973ebed2db00ab6b5be37bf7f7026eea10a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c1d298712c5c3ea2c0294d603cda00c713c367daeedb531b3b9828faf8e5582b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c2b3e56093fa347ac44a6564dc83b18e1731c958bd905686c717ec056b725ce7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c3c54235f5bab1ed193af1a8bf1c53c0ebaaa00d911fbec180e364f4a51f51ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"c9ef03a3c161cff519a0cc77dd1fa8a976c7e422f3986b110e834eef903df89a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"d1b2d1543f80338156eab672e9aabc48a634a2e3aed6e7362f6aeae1408ad8a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"e11b11903d7834d0633cec99c8e016a5437f7fb4b223d336b68e93d4889ded6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"e63bfecd2e80cd34d843b393871c75a9db632999606092b6f2b7879d05c4a6ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"ef690185b55d76436da7c8d08b241833303008c4ac0da413e4ac3bbff2970319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"efdc91261b3dc5be81a96c6a9bdf759132711c9bbb25b59724e796987c7c1da9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"f1229bb403839cad4628996a6410a9af531a5da6f8e4d9788b841ca3b4a79600","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"fe57b1f7e029db16afe6d13d5ca1bd16e0a7f7981e23844e85e0d5c49e7dc6e7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":21.428571428571427,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079074941612855296","occurred_at":"2026-07-20T05:24:28Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":21.428571428571427,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":42,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016028d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9zsl7m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9zsl7m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016028d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9zsl7m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9zsl7m, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"a41a9dd0285ca9f12cb560711358249e2025fbe7cf12cf82a42396190c9042c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079340761836425216","occurred_at":"2026-07-20T23:00:44Z","signature_id":"4127a9187d48928f29732c57d0737caa49f7885f5a800e49e26c0ea8b42fe648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"56f87fb5a807646e254bd4c95e1d7baae5a9e2cc8f9b0f9e1c6f0454d4a97551","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-08T14:15:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax 0xca\nrbx 0x0\nrcx 0x2f2dae3\nrdx 0x0\nrdi 0xc9599f0\nrsi 0x80\nrbp 0x7ffc3cc57520\nrsp 0x7ffc3cc574d8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x7\nr13 0x1\nr14 0xc957640\nr15 0x7\nrip 0x2f2dae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax 0xca\nrbx 0x0\nrcx 0x2f2dae3\nrdx 0x0\nrdi 0xc9599f0\nrsi 0x80\nrbp 0x7ffc3cc57520\nrsp 0x7ffc3cc574d8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x7\nr13 0x1\nr14 0xc957640\nr15 0x7\nrip 0x2f2dae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2f2dae3\nrdx 0x0\nrdi 0xc9599f0\nrsi 0x80\nrbp 0x7ffe78eb87a8\nrsp 0x7ffe78eb8760\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x4c\nr13 0x1\nr14 0xc957640\nr15 0x1\nrip 0x2f2dae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2f2dae3\nrdx 0x0\nrdi 0xc9599f0\nrsi 0x80\nrbp 0x7ffe78eb87a8\nrsp 0x7ffe78eb8760\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x4c\nr13 0x1\nr14 0xc957640\nr15 0x1\nrip 0x2f2dae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"9dfae6e5978c1295fffad1fa285c0f56737b3e5ba630e35f78fc7c230070cc36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","signature_id":"b6f4e9f1f6c7f0e0d12f6274441cf72f39009053ff85d2d81ef74cebb5deb612","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-06","2026-07-13"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-19T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n \u003c*errors.joinError | 0xc000a9c570\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n \u003c*errors.joinError | 0xc000a9c570\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","signature_id":"0d4e6c626552d37165a629be9c672bc072d0e8affb1fa7528cee84809b6aefcf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-19T23:15:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0012881e0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8d968x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\",\n \"name\": \"3888477a-bc90-496f-be5a-132321ab3a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-20T02:16:49.779634198Z\",\n \"endTime\": \"2026-07-20T02:35:49.961113313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0012881e0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8d968x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\",\n \"name\": \"3888477a-bc90-496f-be5a-132321ab3a6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-07-20T02:16:49.779634198Z\",\n \"endTime\": \"2026-07-20T02:35:49.961113313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","signature_id":"c15750545600e820878efaa487132ca680dc610368be4906d55af70ae65160b6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b43e7c6c150476d7400be2924e381be5a30b739ff1a4de2439b70edbc0e574e4","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Network","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidTemplateDeployment\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidTemplateDeployment\",\n \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n \"details\": [\n {\n \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n \"details\": []\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidTemplateDeployment\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidTemplateDeployment\",\n \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n \"details\": [\n {\n \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n \"details\": []\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"075407059f3a5b3d581b963f2098a2be85da109dc5ca5fcc1562bfbb79dd101f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.142857142857143,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-07-14..2026-07-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":8,"prior_last_seen_at":"2026-07-16T09:01:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001a0b110\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001a0b110\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079317230851461120","occurred_at":"2026-07-20T21:27:14Z","signature_id":"ed61138e00b6a5005e4f6c12099e2f0f17e5485a149e9e9870bbab9d9d102348","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}