{"meta":{"start_date":"2026-07-31","end_date":"2026-07-31","start_at":"2026-07-31T00:00:00Z","end_at":"2026-08-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T04:20:53Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":16,"failed_runs":10,"raw_occurrences":70,"matched_occurrences":70,"runs_affected":10},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":47,"runs_affected":4,"failed_runs":4,"run_impact_percent":25,"anchor_occurrences":47,"after_last_push_count":62,"trend_counts":[0,0,0,3,8,4,47],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-22-pb6jrkbs92lk/cluster-nightly-4-22-xssrv8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012790e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xssrv8: failed waiting for cluster=\"cluster-nightly-4-22-xssrv8\" in resourcegroup=\"rg-nightly-4-22-pb6jrkbs92lk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n      \"name\": \"e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:32:11.571189856Z\",\n      \"endTime\": \"2026-07-31T23:51:15.176646708Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-nightly-4-22-pb6jrkbs92lk/cluster-nightly-4-22-xssrv8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012790e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-xssrv8: failed waiting for cluster=\"cluster-nightly-4-22-xssrv8\" in resourcegroup=\"rg-nightly-4-22-pb6jrkbs92lk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n      \"name\": \"e2edd9a8-76f5-47e7-a76b-e6fc6265bb1b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:32:11.571189856Z\",\n      \"endTime\": \"2026-07-31T23:51:15.176646708Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001104360\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f7f6frqr9hfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n      \"name\": \"1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:17:01.726559494Z\",\n      \"endTime\": \"2026-07-31T13:36:11.083035507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001104360\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f7f6frqr9hfl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n      \"name\": \"1b24afca-e6fa-4265-b0d0-15cc4f331d2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:17:01.726559494Z\",\n      \"endTime\": \"2026-07-31T13:36:11.083035507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f78720\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wmqz9jgvp4kr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n      \"name\": \"c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:19:22.344406702Z\",\n      \"endTime\": \"2026-07-31T13:38:31.033150806Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f78720\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wmqz9jgvp4kr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n      \"name\": \"c41955f3-57ba-4bf6-a29d-63eb759e696a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:19:22.344406702Z\",\n      \"endTime\": \"2026-07-31T13:38:31.033150806Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"b83962d63abd4751f1fb3741f6d0003b179d5f5f8ee9ba77b23446f84ffe47bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"0c1126e6d75cdc7e020f2cb29bb01e8e96f7e0b2735c876f1534e5ec6be736d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"552f71aba079936d33b679345d987c6915e1f5d4c7080b6617700d16096fb85b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"5c7e72fb745c231eec7d48bab072bb96a7bdd874d667b8db0a72e1fe40e0714c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"607628fb1634fcb469dfa7b2aaf90c7408c8b6f018f4a45ddf020ec477f5845c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"7d25c20deea3d7f30cb4abd7abf0ed51fce890b92554c5eddf568175636f7041","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"88e1c6360611cb99db8664b8631b557fcf43a4ebd2fa58f417fa155ba9e0338f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"8a98a7e3baf3339560d27a9c17517801a79e44ba4e182a960f8845d43aa0cbd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"91f19c9c4d341e3acfff76110011b76f24e350c1cb330ab3444ebcd6bbfd552d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"96ea83ea0690b8b14850b079a8cba0997c7b78b48c76c1ba7cafc61eb0bed221","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"d0e63e099576bfd4d02da914e850e70f973afbd34ffb54f28fa5373c510d7c2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"dcf71a6679d8d1d79de65a03c8ed12db833efd96b553139459477d2e1cb8821f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e114cbedec1ecc31c1653acce737dc38649c0cf1338d14bda84b086679da4b71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e1b1ede7ff44850fbb3bb80eb24f477565a243eb8affcc66d3a502223e01b271","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"f29993a7dbfecfb6d4c14d8061f6f5b4f50535f04f03fd4995a208bc605cb2ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"154f5ce0edd523ca30c455a745a6548ec024a3fecbf65bf5c3ed075cbcbefded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"22943a83b5a807a6c6ffb231051379e8f3cfba5ae7dfbed35ebe9fc1cc75321e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"25008d95567a88684a2401a50fa214f41e0b84e2d1dec9c6147209f98565d05a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"2f877087d24f1e4e43127361049faeed95382712e0b72bf3191afadfb5f39315","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"48aa84bf3a4ab3ae6e2edfbe0ed7326dbe11e7a129a51125ec3338ac1e04aa37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"4e9036b6f419e800e27e7212dc4f4e9b7a7ecba2dc8927e0c7f9ea029ca257c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"54c200ddcc4b6b4d6b780b2d6fedb782f78d76edab7742cd13cf4c42db5b6346","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"573385bd853245689458f88b5830b3711397b2693998619d3957c84c8fcd9eba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"598d8eb936530f403448bb85cf6470e0108e2d0887675e2c547e501dd8e2698a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"5d3cd084884fae8cfd89c38eeadaecaa371799f087ea96cfc086030755e93359","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"63334c83aea6313567e61807e669f6eaf93ce8a667efae52f6a8a32ea39a7356","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6c5c182039f7672dc075fa72b54b020a27dba8e597655cef85eec4043eabaa40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6ca1e397f6cd8b74e21110b1f5fbce7cdf75978c696708a7a4e2f0d0f28c4eec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"89a89a3a29d9fb04451cf8ae9e348d1106823e4aa50b21a935ec61e75735db22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"906befff4829c721784a271a1ed5b734525fa389cc169bbe54a38a5582736ac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"96b4d73842c958bf6629ec879c25fb07e4c7558543d52db28eb48e259ef8c13a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"9cf910a3bca8d2d36ad62cb309495e5141ee82a6e645bd084cc851c4d118b418","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a1a8244f248eb61e1f2faf3baa67ca04b0bc9681b815e51682f66356757489ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a9fb25a5b04389ff66308052341420205d3ef1800d543533ae64ea0e425d0c94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"adcb8d217761898b3931e51c0af842b6f345b29c2fc2fc186b0c88e3d636f4d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"b01e75249b24e2d6fa4aa72c2300fde23de6d71f37b3e421979785371f7df3b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"b36413c750154c83c94934fe75957298ae57383c5b39c72e5b0211ef615cf651","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"c16325fdee230d6022e4009b6044e73ca1e8a7c8744e735f11ddf531839c9f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"cdf5f5f8f81b6af2194a31a0ee8b9bf30b4a9fa000facb1d01d67199637f56b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"dc6d73660e247263007affc3f4f8bb6b1eee9c82f5b3b07f0613d0203f757e79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"e33d17059d28a9d326e9f2de865ed6ddad82a5c325137c25ff2d33a25a8cfff9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"e67e9411babb49a9fa9afa754913a479d3d1e68a9b54dca6fdd595642cb6d3c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"ea640be7c764230e5b99d89aa2b77d3f2b6d8bb8d7ff99fafcd3dd5fb29d6f10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"eeb9e5277312013361542a3d83aa316edadd54874d2482f1d922aa82a8006029","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"f828b6cf6e4c8d20d69cac8a4dc69f9f0da2dbd01ee9086224275c63d1cafafc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","signature_id":"b0aeb13bb7636b812ad85a1503b71e1488603b540400913f781cd52a11d81b47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083122886733729792","occurred_at":"2026-07-31T09:29:33Z","signature_id":"cccdad4e7c305a79b69a618a299a0aca191ce2b7e21021783762aa34e265d4b0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":12.5,"anchor_occurrences":7,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,2,7],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8a2a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m4rgz4zh45t4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n      \"name\": \"69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:19:20.840846266Z\",\n      \"endTime\": \"2026-07-31T13:38:21.001787978Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8a2a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m4rgz4zh45t4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n      \"name\": \"69ab0ebd-1cc7-4244-b85c-30c06bc68461\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T13:19:20.840846266Z\",\n      \"endTime\": \"2026-07-31T13:38:21.001787978Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df4740\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnfkplvhg68n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\",\n      \"name\": \"3b133ef7-396b-4152-900c-02c4e236373d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T12:02:16.489433823Z\",\n      \"endTime\": \"2026-07-31T12:21:24.280178661Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df4740\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jnfkplvhg68n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3b133ef7-396b-4152-900c-02c4e236373d\",\n      \"name\": \"3b133ef7-396b-4152-900c-02c4e236373d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T12:02:16.489433823Z\",\n      \"endTime\": \"2026-07-31T12:21:24.280178661Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-6wkt8pflljjl/cluster-candidate-4-22-lmtk24 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001514800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-lmtk24: failed waiting for cluster=\"cluster-candidate-4-22-lmtk24\" in resourcegroup=\"rg-candidate-4-22-6wkt8pflljjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n      \"name\": \"0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T12:46:24.136498027Z\",\n      \"endTime\": \"2026-07-31T13:05:25.610298911Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-6wkt8pflljjl/cluster-candidate-4-22-lmtk24 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001514800\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-lmtk24: failed waiting for cluster=\"cluster-candidate-4-22-lmtk24\" in resourcegroup=\"rg-candidate-4-22-6wkt8pflljjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n      \"name\": \"0097c7dd-b1df-4ec0-968a-4af85bbbc3fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T12:46:24.136498027Z\",\n      \"endTime\": \"2026-07-31T13:05:25.610298911Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"e766f3893d259999bf7feb58da887f8fddb7bf6407a6772cd14aea22320aa34c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"1effbddc31f6279209dedbf019d17f5e620f7b5d749048b6663ae3183edf1438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"42a109ad57658f9dd5b2ea83c3874cc37d40e669625c71bfdb720ca6a6021a88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"6d691098b25b913bc371e403e0b7755feb4b06711ad5a28796adcc05df7150c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"7a9ddd001fe58609fa7bc919b8085974ddd5414eca89207c89a656fef92e9af2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"a0b9995e1016cea8831a9346afa65cb0de0ccb6dc63fd78bef195dc758065084","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"aae7e95022d6cbcd106a7aee2159322a5ee8fa7942d388b8298ca95b862dee37","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":25,"anchor_occurrences":4,"after_last_push_count":12,"trend_counts":[0,0,0,1,0,1,4],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":13,"prior_last_seen_at":"2026-07-22T23:00:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bgmptg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb8ca0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-lmdkg.apps.aro.p1l0j8m7f2t6o2v.wzf5.canadacentral.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-n7l2w2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a001a0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-hhdrv.apps.aro.n7l8a2p4i7p3w7g.8d7k.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84940\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:187]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bfhpb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e84940\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fsb9t.apps.aro.m0y5v1t7g5f7f6f.6ugv.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083208396000989184","occurred_at":"2026-07-31T15:09:20Z","signature_id":"90d4fdfb969a770390c563f242d33f0daa3473e0b30ba9027c45b262814b26d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083172285048426496","occurred_at":"2026-07-31T12:45:50Z","signature_id":"f3674ccfd569851653672f515b42e2695a6f9f20f7a1fdd0153289b8e1933ea3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083129615265566720","occurred_at":"2026-07-31T09:56:17Z","signature_id":"0331cad4f2a58b011ffbb47661db7de7b69b057a6cf4c77587a1708b2b88ab94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083096246586707968","occurred_at":"2026-07-31T07:43:41Z","signature_id":"b3787027bfe54fcb1e26887ea60cdb16b0f24ce43553bbde8fc9ce9b610a6a9b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":18.75,"anchor_occurrences":3,"after_last_push_count":14,"trend_counts":[0,0,0,1,1,0,3],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-23T12:25:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.25,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":14,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00137c558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00137c558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\", caused by: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-bdx9lr\" in resourcegroup=\"rg-np-skip-minor-bdx9lr-s76fpj96mgg6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-bdx9lr in resource group rg-np-skip-minor-bdx9lr-s76fpj96mgg6, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010ad1d0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\", caused by: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-dg-minor-jdzsdl\" in resourcegroup=\"rg-np-dg-minor-jdzsdl-w6gs52xscdrl\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-dg-minor-jdzsdl in resource group rg-np-dg-minor-jdzsdl-w6gs52xscdrl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e04108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e04108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\", caused by: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8hnjmfxdf5l4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster shoebox-hcp-cluster in resource group shoebox-cluster-8hnjmfxdf5l4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"104e9b9a5b09b9083011ea98bb3e00c0b9a6c2cf07df0e0d78da950283d3b4ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"31b06f078f50236696796c5d7f165517cbc573486d9f7fc2bf92807bf3599f7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083076738530676736","occurred_at":"2026-07-31T06:26:10Z","signature_id":"423dbfb7ba3def13bdce0df4b3635387513bf45addadf90a3c38c3ed7dac329a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":12.5,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-21T05:13:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083175015817154560","occurred_at":"2026-07-31T12:56:41Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.25,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":33,"prior_last_seen_at":"2026-07-21T05:13:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ad30b0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-84d9mn7wrnll\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-84d9mn7wrnll, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pqzcmf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000da2f60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pqzcmf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: failed waiting for cluster=\"np-version-upgrade-cluster-pqzcmf\" in resourcegroup=\"rg-np-version-upgrade-pqzcmf-mg822cfcxtc6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pqzcmf in resource group rg-np-version-upgrade-pqzcmf-mg822cfcxtc6, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"621048a5aac4e75e14d77235942749138b536b5328379c9bef941f0c7f0665f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083159875340210176","occurred_at":"2026-07-31T11:56:32Z","signature_id":"723f7b27a9cb71599870027d12663525adfe554e9b07936872529641bf13f522","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.25,"anchor_occurrences":1,"after_last_push_count":18,"trend_counts":[0,0,1,1,1,5,1],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-20"],"prior_runs_affected":15,"prior_last_seen_at":"2026-07-23T07:09:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002930018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n      \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002930018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-x6fgs25whrrl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"name\": \"890b1b13-0070-4aa1-9e40-2f45c5b4dad5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T23:22:53.3740224Z\",\n      \"endTime\": \"2026-07-31T23:41:54.935883407Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2083326864956329984","occurred_at":"2026-07-31T23:00:05Z","signature_id":"4072a5302e0e12ad0124844ab5b66b17d835528aa52fe99adcf4fce54e5c47d5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.25,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-07-25..2026-07-31 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093af20\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00093af20\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jtfcvfq7vbjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtfcvfq7vbjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083009755474825216","occurred_at":"2026-07-31T02:00:00Z","signature_id":"033938777cbf5c901d58fef72baead815b4f5bdf96d12c88b67aa03ba6ca5fdf","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}