{"meta":{"start_date":"2026-07-27","end_date":"2026-08-02","start_at":"2026-07-27T00:00:00Z","end_at":"2026-08-03T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T02:00:04Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":70,"failed_runs":33,"raw_occurrences":162,"matched_occurrences":160,"runs_affected":31},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":91,"runs_affected":10,"failed_runs":10,"run_impact_percent":14.285714285714286,"anchor_occurrences":91,"after_last_push_count":91,"trend_counts":[0,0,3,8,4,47,0,29],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xjxxj2swjnz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"name\": \"df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:04.574656397Z\",\n      \"endTime\": \"2026-08-02T10:40:12.775068231Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xjxxj2swjnz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"name\": \"df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:04.574656397Z\",\n      \"endTime\": \"2026-08-02T10:40:12.775068231Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-nhx2wp\" with version 4.19.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097aa00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-nhx2wp: failed waiting for cluster=\"cluster-zstream-4-19-nhx2wp\" in resourcegroup=\"rg-zstream-upgrade-4-19-rn966m5vc2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"name\": \"3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:38.264467464Z\",\n      \"endTime\": \"2026-08-02T10:39:42.477082291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-nhx2wp\" with version 4.19.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097aa00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-nhx2wp: failed waiting for cluster=\"cluster-zstream-4-19-nhx2wp\" in resourcegroup=\"rg-zstream-upgrade-4-19-rn966m5vc2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"name\": \"3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:38.264467464Z\",\n      \"endTime\": \"2026-08-02T10:39:42.477082291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b8tzdmst7fvv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"name\": \"d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:21.5378839Z\",\n      \"endTime\": \"2026-08-02T09:53:39.994858773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b8tzdmst7fvv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"name\": \"d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:21.5378839Z\",\n      \"endTime\": \"2026-08-02T09:53:39.994858773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"051cf6ecf727851a70ff0cd34517780b1e9e358968115dffad4d43429bd5ffc8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"052ebe2c3c97bb9a568990baa076d6e2109f823a9569c9d5902807103f1e7ba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"16c7003b3e534c2db0acd58bb74f730ca9c0be4f2219f61b2816b7c3c71ee0d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"1a30293f03d5bebc67ec881255dfa940c8186d7d08111f8bbe98b67241d9d72b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"1bc26a0d8ae0ce7fe6f1aba0d8fc87d14bcd6a2b503d2c98efa3e089d66e2292","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"2e944f7dcda11dea00030f48665f964060e872901fa8be59db56f811abce6888","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"3428e446bfa285f9bf1d24feb83c705e1ee307dcfb4a3915e67574e287931754","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"38751658fb583189a0984e0db3eee96ce7a1813b6575d5270ee70cb61a4efdaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"4dac8a5c315ca81423f117e3fca48c1fa722a6c62acb39a85e9651b3ff831f22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"52038d3d19bf3139552e5ab7d7020bef9adf291bda0c9e9fd0263000496ac854","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"5fb90d3ae62f48b86e0520e6fbb53bb0d6a7df71901e6e1dd5d13262315140b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"60f68294f0de255f611b75fe84b7d0107a776535c9c347d611014215381bca2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"6767756089139aab74b8a0e4afea6f3d529957e75af8bbec4c5d9f48607a5b54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"73397df6b0d9f6da82d4449e3fa8ea2eccb205f8dd99d6726b907bd5b53a5a64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"758100d66a6991647c1f198297b30ee84054e8e323d4bb3a4dc793e5f36333c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"8fa8b02b7d8b636eef0c10e97ce96c5b5d84f42b39fc1ad41b2f7c1c3750d7de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"99f53693e4f66cf8cd29dad172acb0d2eb966d3db7b113aaa865daef312a6804","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"9b8250b9774580074602a315c723a89e946d1b5bf10a47c683301668aec3a81d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"a4aad4c43570be1a3f9d2d924b3f6f5e8247e1935408a40920e54c59ba43a538","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"ae05d6e06d1b0cf99d32c9111db86448d9f13d61f2c47276a1d08eefc967fc14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"af673c3f3931c89c69892df1b525596bf2fb5688460fa909b7990682782a2796","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"c0a31610c203d33384a042a830ae08ce812df9f1fb4d5c64c588207e18c01528","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"c2862f8d2af906875d80375e9fa90f8ff9bc02e2357e85ccfe0589b8d49ed29e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"cb6eb456a0df8f447cc45568b17d4fc671353f4f3886840340943f1ee9cd92c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"ea5855a74ab16b441b7b8667c71eb933594848719f04233ef7b8daee27e38120","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"eef1f7d9c831ab92e3e3bd5d202952d2fed84be97e70dc93fbbbc52dfe7294da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"f9d091a59a7901e6df23ca61e4790b655188be668b236e3780226d657a1365e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"fcf93f141a243945f67f44b14e73f777d9be669780336c2e9993260d931f7481","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"fff0e5b06a86af9a09ba19f4ff37b1e7c499c4333027af36c522fa247ad2f3c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"b83962d63abd4751f1fb3741f6d0003b179d5f5f8ee9ba77b23446f84ffe47bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"0c1126e6d75cdc7e020f2cb29bb01e8e96f7e0b2735c876f1534e5ec6be736d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"552f71aba079936d33b679345d987c6915e1f5d4c7080b6617700d16096fb85b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"5c7e72fb745c231eec7d48bab072bb96a7bdd874d667b8db0a72e1fe40e0714c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"607628fb1634fcb469dfa7b2aaf90c7408c8b6f018f4a45ddf020ec477f5845c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"7d25c20deea3d7f30cb4abd7abf0ed51fce890b92554c5eddf568175636f7041","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"88e1c6360611cb99db8664b8631b557fcf43a4ebd2fa58f417fa155ba9e0338f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"8a98a7e3baf3339560d27a9c17517801a79e44ba4e182a960f8845d43aa0cbd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"91f19c9c4d341e3acfff76110011b76f24e350c1cb330ab3444ebcd6bbfd552d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"96ea83ea0690b8b14850b079a8cba0997c7b78b48c76c1ba7cafc61eb0bed221","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"d0e63e099576bfd4d02da914e850e70f973afbd34ffb54f28fa5373c510d7c2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"dcf71a6679d8d1d79de65a03c8ed12db833efd96b553139459477d2e1cb8821f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e114cbedec1ecc31c1653acce737dc38649c0cf1338d14bda84b086679da4b71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e1b1ede7ff44850fbb3bb80eb24f477565a243eb8affcc66d3a502223e01b271","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"f29993a7dbfecfb6d4c14d8061f6f5b4f50535f04f03fd4995a208bc605cb2ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"154f5ce0edd523ca30c455a745a6548ec024a3fecbf65bf5c3ed075cbcbefded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"22943a83b5a807a6c6ffb231051379e8f3cfba5ae7dfbed35ebe9fc1cc75321e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"25008d95567a88684a2401a50fa214f41e0b84e2d1dec9c6147209f98565d05a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"2f877087d24f1e4e43127361049faeed95382712e0b72bf3191afadfb5f39315","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"48aa84bf3a4ab3ae6e2edfbe0ed7326dbe11e7a129a51125ec3338ac1e04aa37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"4e9036b6f419e800e27e7212dc4f4e9b7a7ecba2dc8927e0c7f9ea029ca257c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"54c200ddcc4b6b4d6b780b2d6fedb782f78d76edab7742cd13cf4c42db5b6346","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"573385bd853245689458f88b5830b3711397b2693998619d3957c84c8fcd9eba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"598d8eb936530f403448bb85cf6470e0108e2d0887675e2c547e501dd8e2698a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"5d3cd084884fae8cfd89c38eeadaecaa371799f087ea96cfc086030755e93359","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"63334c83aea6313567e61807e669f6eaf93ce8a667efae52f6a8a32ea39a7356","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6c5c182039f7672dc075fa72b54b020a27dba8e597655cef85eec4043eabaa40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6ca1e397f6cd8b74e21110b1f5fbce7cdf75978c696708a7a4e2f0d0f28c4eec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"89a89a3a29d9fb04451cf8ae9e348d1106823e4aa50b21a935ec61e75735db22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"906befff4829c721784a271a1ed5b734525fa389cc169bbe54a38a5582736ac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"96b4d73842c958bf6629ec879c25fb07e4c7558543d52db28eb48e259ef8c13a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"9cf910a3bca8d2d36ad62cb309495e5141ee82a6e645bd084cc851c4d118b418","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a1a8244f248eb61e1f2faf3baa67ca04b0bc9681b815e51682f66356757489ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a9fb25a5b04389ff66308052341420205d3ef1800d543533ae64ea0e425d0c94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"adcb8d217761898b3931e51c0af842b6f345b29c2fc2fc186b0c88e3d636f4d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"b01e75249b24e2d6fa4aa72c2300fde23de6d71f37b3e421979785371f7df3b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"b36413c750154c83c94934fe75957298ae57383c5b39c72e5b0211ef615cf651","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"c16325fdee230d6022e4009b6044e73ca1e8a7c8744e735f11ddf531839c9f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"cdf5f5f8f81b6af2194a31a0ee8b9bf30b4a9fa000facb1d01d67199637f56b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"dc6d73660e247263007affc3f4f8bb6b1eee9c82f5b3b07f0613d0203f757e79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"e33d17059d28a9d326e9f2de865ed6ddad82a5c325137c25ff2d33a25a8cfff9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"e67e9411babb49a9fa9afa754913a479d3d1e68a9b54dca6fdd595642cb6d3c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"ea640be7c764230e5b99d89aa2b77d3f2b6d8bb8d7ff99fafcd3dd5fb29d6f10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"eeb9e5277312013361542a3d83aa316edadd54874d2482f1d922aa82a8006029","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"f828b6cf6e4c8d20d69cac8a4dc69f9f0da2dbd01ee9086224275c63d1cafafc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","signature_id":"b0aeb13bb7636b812ad85a1503b71e1488603b540400913f781cd52a11d81b47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","signature_id":"cccdad4e7c305a79b69a618a299a0aca191ce2b7e21021783762aa34e265d4b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"29160597ae1b571b364c2eeabf2769302264c37cc8a76086b92bf9b2a5805197","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"53759798de1b1cbea6d270261d69b47e856f4593abcfe1c2d0bc5850c3342479","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"8ec59ee03d15896018b5346f71607e10a2e74944efd51bc21c29d9053c3d03d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082675949039521792","occurred_at":"2026-07-30T03:53:35Z","signature_id":"5b02f0fbf54d86701870a336fa2e752001e95ededdf41894d945e1f818c12826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"3320b7b421abc7495f3d38d4c54d92df2ee5569a5b7507340497c93c97a16b2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"83a466dabf4df209fbbc153c92389d7b33d53abfa5d0ea52538017c262aa8d7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"89ec90a05854a96e4314d56d7b2103aca5da8c5827d4d0912eba0a8b49920eb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"985863ee16558d554939b5436eded2bc5d8e28deece7209755185db265ecdbd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"bf6b529a259dfb87e6ef3b60ae798f6e0f770f84da51d8163a2c8622149afbd5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"c46ba0cb07eeb66c683f5e69d56a31d8f06a3223bda86d64c6cbafa4257b8b22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"eee34cb496c5a8c7be29d16abaf5d0a666a957636e08b1c93061315751f8b52f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082285143296315392","occurred_at":"2026-07-29T02:00:39Z","signature_id":"fe1ab3d15d93157c8f567e9b49bfe94cb77c39e9d6c00f3889916cd9f2f74c0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082224410923634688","occurred_at":"2026-07-28T21:59:20Z","signature_id":"1a5451e74c4cea9da44b775bbc926309f8a51d9248b6bd83be3b0d10aec09a4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082224410923634688","occurred_at":"2026-07-28T21:59:20Z","signature_id":"919691f22fce5bd8e48557f974a513f0e17380a401bace8f955150297ef316d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082224410923634688","occurred_at":"2026-07-28T21:59:20Z","signature_id":"945ea106b562eac00665e926651a97aa592ae34a1436ab3bb7a85844779ed98a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":15,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.714285714285714,"anchor_occurrences":15,"after_last_push_count":15,"trend_counts":[0,0,0,0,2,7,0,6],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-thgbrpsnmsdd/cluster-candidate-4-21-vvlhtd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vvlhtd: failed waiting for cluster=\"cluster-candidate-4-21-vvlhtd\" in resourcegroup=\"rg-candidate-4-21-thgbrpsnmsdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"name\": \"cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:40:52.659578391Z\",\n      \"endTime\": \"2026-08-02T10:00:01.276872508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-thgbrpsnmsdd/cluster-candidate-4-21-vvlhtd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vvlhtd: failed waiting for cluster=\"cluster-candidate-4-21-vvlhtd\" in resourcegroup=\"rg-candidate-4-21-thgbrpsnmsdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"name\": \"cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:40:52.659578391Z\",\n      \"endTime\": \"2026-08-02T10:00:01.276872508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-p79b4rkhtstx/cluster-candidate-4-20-qrm8jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qrm8jx: failed waiting for cluster=\"cluster-candidate-4-20-qrm8jx\" in resourcegroup=\"rg-candidate-4-20-p79b4rkhtstx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"name\": \"8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:10.119552619Z\",\n      \"endTime\": \"2026-08-02T09:54:10.020290658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-p79b4rkhtstx/cluster-candidate-4-20-qrm8jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qrm8jx: failed waiting for cluster=\"cluster-candidate-4-20-qrm8jx\" in resourcegroup=\"rg-candidate-4-20-p79b4rkhtstx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"name\": \"8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:10.119552619Z\",\n      \"endTime\": \"2026-08-02T09:54:10.020290658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003fe7c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q7k5fm2cgm92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"name\": \"66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:36:47.470711734Z\",\n      \"endTime\": \"2026-08-02T09:55:49.913729058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003fe7c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q7k5fm2cgm92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"name\": \"66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:36:47.470711734Z\",\n      \"endTime\": \"2026-08-02T09:55:49.913729058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"05a2a50e8a6091e0075f3d39075b8f373088d19d53d30f09701a323b71da89a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"403cf24078d9fc5ef87306cc4f3947de25eca13b005e2ccbead7322773601632","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"6d309d366f045b03e648f62a1615aad3e21f681888bbee67c72198bd0dd762f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"ab42de5f46497a16c405d24d5aa9cde06b8fea1cc967183d5966c2a2ab8f84ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"b40a1fb2ac4a8cba9f1d30b8d5fea9aa9626d88c7b7d0cc04bc9194ba36da5bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"c59f01db4e6f5f62a207e37ba88d75f58c2705406f4799ff08454e16bd714d9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e766f3893d259999bf7feb58da887f8fddb7bf6407a6772cd14aea22320aa34c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"1effbddc31f6279209dedbf019d17f5e620f7b5d749048b6663ae3183edf1438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"42a109ad57658f9dd5b2ea83c3874cc37d40e669625c71bfdb720ca6a6021a88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6d691098b25b913bc371e403e0b7755feb4b06711ad5a28796adcc05df7150c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"7a9ddd001fe58609fa7bc919b8085974ddd5414eca89207c89a656fef92e9af2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a0b9995e1016cea8831a9346afa65cb0de0ccb6dc63fd78bef195dc758065084","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"aae7e95022d6cbcd106a7aee2159322a5ee8fa7942d388b8298ca95b862dee37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"b89c7ac67f41ad70087c234e520a4a2da57fc04986a62c8408dfe0724611b225","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"f329a6520f95944717d8a48e410a87d34976660aacc46f3d9e2e9552341a68de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":11,"runs_affected":9,"failed_runs":9,"run_impact_percent":12.857142857142858,"anchor_occurrences":11,"after_last_push_count":20,"trend_counts":[0,1,1,1,5,1,0,2],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-20"],"prior_runs_affected":17,"prior_last_seen_at":"2026-07-23T07:09:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24a00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9ngxf9zlhblm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"name\": \"3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T23:16:20.627905758Z\",\n      \"endTime\": \"2026-08-02T23:35:28.37866899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24a00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9ngxf9zlhblm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"name\": \"3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T23:16:20.627905758Z\",\n      \"endTime\": \"2026-08-02T23:35:28.37866899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00214e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-l54jll2tvrwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"name\": \"0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T02:20:43.022860317Z\",\n      \"endTime\": \"2026-08-02T02:39:48.638662525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00214e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-l54jll2tvrwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"name\": \"0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T02:20:43.022860317Z\",\n      \"endTime\": \"2026-08-02T02:39:48.638662525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002930018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n      \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002930018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n      \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084051647016734720","occurred_at":"2026-08-02T23:00:07Z","signature_id":"9f0e17695dbb84bcb7660491c012000cf531902427748b6227ba96eff615d5d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083734553368727552","occurred_at":"2026-08-02T02:00:06Z","signature_id":"d45338114907e1f99cd34e1b097973074d907ac30b280168e474872bc1831e78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"4072a5302e0e12ad0124844ab5b66b17d835528aa52fe99adcf4fce54e5c47d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082899684623388672","occurred_at":"2026-07-30T18:42:37Z","signature_id":"7d2bcaa645afc4ae9592bf1820a2171f5d94eaca375fb1bd194afd110b7d5809","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","signature_id":"1ab6fbef233dddf10f89d2a7fc671a3b1aaa7128d75d931d0bc6f823883a1e94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"39d0da823060c3256ca361233cddb71ef25ed0003a89f41041e8c0a328409a12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"6ac211ba10e267bd185623c33c1580de83901464fb1a55e5f6787ef19f4c86bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"b52edbdd27541815677acf3313e8a385ea427ab0cb29141a18288e99da41fcef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082426114898137088","occurred_at":"2026-07-29T11:20:50Z","signature_id":"fdc5d2be0d25f697e55b562829775e172533b19f5057a9135b273313e4c97e0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082100651952705536","occurred_at":"2026-07-28T13:47:33Z","signature_id":"09b2268658174d38c77b89f590b6d7237de4277cb9d994640620567ed65ca678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2081691944936804352","occurred_at":"2026-07-27T10:43:30Z","signature_id":"16fc271ab82f5380f7834b73fed953f82ead95096fdf9f5d0795abf2b7eba540","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":11,"runs_affected":5,"failed_runs":5,"run_impact_percent":7.142857142857143,"anchor_occurrences":11,"after_last_push_count":43,"trend_counts":[0,0,0,4,4,3,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00137c558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00137c558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e04108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e04108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"104e9b9a5b09b9083011ea98bb3e00c0b9a6c2cf07df0e0d78da950283d3b4ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"31b06f078f50236696796c5d7f165517cbc573486d9f7fc2bf92807bf3599f7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"423dbfb7ba3def13bdce0df4b3635387513bf45addadf90a3c38c3ed7dac329a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082899684623388672","occurred_at":"2026-07-30T18:42:37Z","signature_id":"409703503e6aea948ba52c3c5eb1f2c0d8db728a3160f7b9df9d614dc003e6cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"2c788d8a3815e3876197d8f2e7d6ddac61b59269c848b674bc458f1031ac1846","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"a7ae58b0c35201d9638220ac10a4f2fa47986d5db40a05cdc9220891d4f429e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"c23160afc432e787bd9c17aa2fcc0f9f45b004d10323c162c9f24a2573785276","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"7e2a79de725776165b8aa00972dd0a55650a63cb3758764c653899700a45bcc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"bc1dfc2c55c0166aaf97f354cd93cc3c56076c1026d4eee5e529342b2fbe5534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"dbef0570edb7869dcc70fa2607e08962e0ef53d745127ecc46f504d82360bd8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082259111117852672","occurred_at":"2026-07-29T00:17:13Z","signature_id":"55c6bf9689c47e7e895ffff2e4c5a16c064dfdccbb11b42f90d9359f0f01c1f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":8.571428571428571,"anchor_occurrences":6,"after_last_push_count":15,"trend_counts":[0,0,1,1,0,3,0,1],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":15,"prior_last_seen_at":"2026-07-23T12:25:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082224410923634688","occurred_at":"2026-07-28T21:59:20Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":8.571428571428571,"anchor_occurrences":6,"after_last_push_count":12,"trend_counts":[0,0,1,0,1,4,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":13,"prior_last_seen_at":"2026-07-22T23:00:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84940\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84940\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083208396000989184","occurred_at":"2026-07-31T15:09:20Z","signature_id":"90d4fdfb969a770390c563f242d33f0daa3473e0b30ba9027c45b262814b26d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083172285048426496","occurred_at":"2026-07-31T12:45:50Z","signature_id":"f3674ccfd569851653672f515b42e2695a6f9f20f7a1fdd0153289b8e1933ea3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083129615265566720","occurred_at":"2026-07-31T09:56:17Z","signature_id":"0331cad4f2a58b011ffbb47661db7de7b69b057a6cf4c77587a1708b2b88ab94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083096246586707968","occurred_at":"2026-07-31T07:43:41Z","signature_id":"b3787027bfe54fcb1e26887ea60cdb16b0f24ce43553bbde8fc9ce9b610a6a9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082641152187568128","occurred_at":"2026-07-30T01:34:23Z","signature_id":"0d1a1089eb774ffa3548638344a1e2b0fe4f14bbefe85bfa532befdca533fb72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2081922822891900928","occurred_at":"2026-07-28T02:00:55Z","signature_id":"ffccf80fa8056cee82f3e7e67eb3e21a25a47265b97b752f09dfbeec641a0da4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.857142857142857,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,2,2,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca1f40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca1f40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e79e40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e79e40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082647614926360576","occurred_at":"2026-07-30T02:00:59Z","signature_id":"664e785e76444d94ea714dbd3345cd8c8aa34df77b19996aacc8a85ca28e6c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082647614926360576","occurred_at":"2026-07-30T02:00:59Z","signature_id":"e3f0e8d101926d54b52273c737c587f0ca58a59d12fbb0b58cb0f281747a8b4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","signature_id":"3d8c987c527570b02284ae5b1dd3553b6cc602b543deeb4c7abc806b1b8eb8c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","signature_id":"a6b013ff1d8bdd34482eff2d5f6a091650cd8c4a6b2f95151702b83e709e2576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4285714285714286,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,4,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-09T15:23:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00294ae10\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00294ae10\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a941e0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-d45wt7\" resourcegroup=\"rg-np-version-upgrade-d45wt7-r5nbl95ln47b\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-d45wt7 in resource group rg-np-version-upgrade-d45wt7-r5nbl95ln47b, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a941e0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-d45wt7\" resourcegroup=\"rg-np-version-upgrade-d45wt7-r5nbl95ln47b\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-d45wt7 in resource group rg-np-version-upgrade-d45wt7-r5nbl95ln47b, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-21 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001096720\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-m98bwq\" resourcegroup=\"rg-np-version-upgrade-m98bwq-6wlkt7dwt5st\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-m98bwq in resource group rg-np-version-upgrade-m98bwq-6wlkt7dwt5st, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-21 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001096720\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-m98bwq\" resourcegroup=\"rg-np-version-upgrade-m98bwq-6wlkt7dwt5st\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-m98bwq in resource group rg-np-version-upgrade-m98bwq-6wlkt7dwt5st, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"3f36455f13241999b39b257f5b49cb419277fb42d5614d28b4e671d9055c4d03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"6f5f5908ee3b39d280f7e0e40faf8eaef29dd4317f23980e3578da9280e33c74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"b092477ead04bebcbc3e54ba7df3b6a17e380d8467de8e0d18c3adfe414687c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","signature_id":"e4a8538450a0681886da6c98e27e84371e6ba2092979838ec812325cc8162e03","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":4.285714285714286,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,2,0,1],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":7,"prior_last_seen_at":"2026-07-21T05:13:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.857142857142857,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,2,0,1],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":34,"prior_last_seen_at":"2026-07-21T05:13:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001412750\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sf8fnq7ckvqz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sf8fnq7ckvqz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001412750\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sf8fnq7ckvqz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sf8fnq7ckvqz, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","signature_id":"e04dbd31283ca9365e3d1599ffde4d002fd0443462663b99ce14916d32dabe89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"621048a5aac4e75e14d77235942749138b536b5328379c9bef941f0c7f0665f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"723f7b27a9cb71599870027d12663525adfe554e9b07936872529641bf13f522","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.857142857142857,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,1,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093af20\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093af20\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004351e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-q9gpdmtglj9j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q9gpdmtglj9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004351e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-q9gpdmtglj9j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q9gpdmtglj9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083009755474825216","occurred_at":"2026-07-31T02:00:00Z","signature_id":"033938777cbf5c901d58fef72baead815b4f5bdf96d12c88b67aa03ba6ca5fdf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082964456261816320","occurred_at":"2026-07-30T23:00:00Z","signature_id":"cc554efcb1bebe7d990883ab94bead887ef4c10bf0225aa8d891690288360dae","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9095e6a1d01ab4218426574bc1b1f4094fa1640613eff3cdbf58f78fedc6a291","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:411]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:410 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:411]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:410 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","signature_id":"7a1811ca6cb73bd5963ea482dfbb3e212685b616d35f1d264dfe7bdc88eea013","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-pxnh4n9clmln\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001ade210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-pxnh4n9clmln\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001ade210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","signature_id":"b2240ed412ce4ac7629bcdff474ee13e9115652c8ff23691e840266cdb7e0f86","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4285714285714286,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-19T17:04:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"a17e2efd2f02a984b507dcc724ffb9c4f854ad0a7bf6c1ddd6785c1d3ad1414a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b8223127140c833357088cdafd7290dc7db01297e09b7695d10e69e7ef3961f2","failure_pattern":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.4285714285714286,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-23T00:28:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00296f170\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00296f170\u003e: \n    add password after 2 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2081560427556245504","occurred_at":"2026-07-27T02:00:54Z","signature_id":"0624a3e94ebb38920211389a774e8d2b1c97830ab07b198dac59b968e389dee6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}