{"meta":{"start_date":"2026-08-12","end_date":"2026-08-12","start_at":"2026-08-12T00:00:00Z","end_at":"2026-08-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T03:52:36Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":9,"raw_occurrences":24,"matched_occurrences":23,"runs_affected":8},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":30.76923076923077,"anchor_occurrences":5,"after_last_push_count":124,"trend_counts":[4,0,0,0,0,0,5],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-03"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-lpkbml642d9f/cluster-candidate-4-21-dlhpq7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4aa00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-dlhpq7: failed waiting for cluster=\"cluster-candidate-4-21-dlhpq7\" in resourcegroup=\"rg-candidate-4-21-lpkbml642d9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c02e1e85-e228-48c4-9f6f-2945951a5ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c02e1e85-e228-48c4-9f6f-2945951a5ba4\",\n      \"name\": \"c02e1e85-e228-48c4-9f6f-2945951a5ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:49:52.442801309Z\",\n      \"endTime\": \"2026-08-13T01:08:55.002562177Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-lpkbml642d9f/cluster-candidate-4-21-dlhpq7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4aa00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-dlhpq7: failed waiting for cluster=\"cluster-candidate-4-21-dlhpq7\" in resourcegroup=\"rg-candidate-4-21-lpkbml642d9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c02e1e85-e228-48c4-9f6f-2945951a5ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/c02e1e85-e228-48c4-9f6f-2945951a5ba4\",\n      \"name\": \"c02e1e85-e228-48c4-9f6f-2945951a5ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:49:52.442801309Z\",\n      \"endTime\": \"2026-08-13T01:08:55.002562177Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-cb87h255nt86/cluster-candidate-4-22-6jhc69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001408b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-6jhc69: failed waiting for cluster=\"cluster-candidate-4-22-6jhc69\" in resourcegroup=\"rg-candidate-4-22-cb87h255nt86\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2b392c9d-db40-489a-9678-f62654ca0bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2b392c9d-db40-489a-9678-f62654ca0bba\",\n      \"name\": \"2b392c9d-db40-489a-9678-f62654ca0bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:50:23.843131255Z\",\n      \"endTime\": \"2026-08-13T01:09:25.008427202Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-cb87h255nt86/cluster-candidate-4-22-6jhc69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001408b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-6jhc69: failed waiting for cluster=\"cluster-candidate-4-22-6jhc69\" in resourcegroup=\"rg-candidate-4-22-cb87h255nt86\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2b392c9d-db40-489a-9678-f62654ca0bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2b392c9d-db40-489a-9678-f62654ca0bba\",\n      \"name\": \"2b392c9d-db40-489a-9678-f62654ca0bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:50:23.843131255Z\",\n      \"endTime\": \"2026-08-13T01:09:25.008427202Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e902c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6qrgrbqbsrdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/463bb008-cd8d-4151-9e0e-f5a642ef319d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/463bb008-cd8d-4151-9e0e-f5a642ef319d\",\n      \"name\": \"463bb008-cd8d-4151-9e0e-f5a642ef319d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T13:28:39.937188776Z\",\n      \"endTime\": \"2026-08-12T13:47:44.080709237Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e902c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6qrgrbqbsrdb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/463bb008-cd8d-4151-9e0e-f5a642ef319d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/463bb008-cd8d-4151-9e0e-f5a642ef319d\",\n      \"name\": \"463bb008-cd8d-4151-9e0e-f5a642ef319d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T13:28:39.937188776Z\",\n      \"endTime\": \"2026-08-12T13:47:44.080709237Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087690481667411968","occurred_at":"2026-08-12T23:59:32Z","signature_id":"042f189bbf8395fa3ab89a9b8030f4009e521e57b437602b2c929482c6f3e2d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087690481667411968","occurred_at":"2026-08-12T23:59:32Z","signature_id":"29dca3743e17072fd4f23b9fe74b43de286a9f6e63806d3f2ec69887696ab24e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"45aa7a7bd3e52102fe5bb126d10b1a4e5b53ca12419750d169a78ae5d15a183a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087461270386118656","occurred_at":"2026-08-12T08:48:44Z","signature_id":"5c42f237bddb79699f4e61ea9cf12c470164c4dfd9b583923308163ddd5d0385","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087424873520435200","occurred_at":"2026-08-12T06:24:06Z","signature_id":"b6e22c7a7303e3611fb355549ee999321de8ea9642b9b7c0ea962f4faed15f5e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":30.76923076923077,"anchor_occurrences":4,"after_last_push_count":25,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":25,"prior_last_seen_at":"2026-08-05T23:00:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.494s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.494s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087567814196989952","occurred_at":"2026-08-12T15:52:06Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527518214033408","occurred_at":"2026-08-12T13:11:59Z","signature_id":"f212c4f8b46003a99564acd7a8782d23474787261609c177238c220a59fe691d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087424873520435200","occurred_at":"2026-08-12T06:24:06Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-drnzfn9bp759/cluster-candidate-5-0-bmpjt4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2cec0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-bmpjt4: failed waiting for cluster=\"cluster-candidate-5-0-bmpjt4\" in resourcegroup=\"rg-candidate-5-0-drnzfn9bp759\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fb97883e-2c9e-417b-b986-1c06da54b8eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fb97883e-2c9e-417b-b986-1c06da54b8eb\",\n      \"name\": \"fb97883e-2c9e-417b-b986-1c06da54b8eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:02:33.71918951Z\",\n      \"endTime\": \"2026-08-12T14:21:35.18215595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-drnzfn9bp759/cluster-candidate-5-0-bmpjt4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2cec0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-bmpjt4: failed waiting for cluster=\"cluster-candidate-5-0-bmpjt4\" in resourcegroup=\"rg-candidate-5-0-drnzfn9bp759\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fb97883e-2c9e-417b-b986-1c06da54b8eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fb97883e-2c9e-417b-b986-1c06da54b8eb\",\n      \"name\": \"fb97883e-2c9e-417b-b986-1c06da54b8eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:02:33.71918951Z\",\n      \"endTime\": \"2026-08-12T14:21:35.18215595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mnfpn75s5xrg/cluster-candidate-4-21-jf7dc8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000168b80\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jf7dc8: failed waiting for cluster=\"cluster-candidate-4-21-jf7dc8\" in resourcegroup=\"rg-candidate-4-21-mnfpn75s5xrg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2728bd8d-0534-4c20-ab83-203edf6c1838\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2728bd8d-0534-4c20-ab83-203edf6c1838\",\n      \"name\": \"2728bd8d-0534-4c20-ab83-203edf6c1838\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:00:50.220066247Z\",\n      \"endTime\": \"2026-08-12T14:19:55.174175712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mnfpn75s5xrg/cluster-candidate-4-21-jf7dc8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000168b80\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-jf7dc8: failed waiting for cluster=\"cluster-candidate-4-21-jf7dc8\" in resourcegroup=\"rg-candidate-4-21-mnfpn75s5xrg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2728bd8d-0534-4c20-ab83-203edf6c1838\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2728bd8d-0534-4c20-ab83-203edf6c1838\",\n      \"name\": \"2728bd8d-0534-4c20-ab83-203edf6c1838\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:00:50.220066247Z\",\n      \"endTime\": \"2026-08-12T14:19:55.174175712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001590160\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-pctqk9m4626n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e8d822d-2e8a-489a-bdca-1ae43bb9700c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e8d822d-2e8a-489a-bdca-1ae43bb9700c\",\n      \"name\": \"1e8d822d-2e8a-489a-bdca-1ae43bb9700c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T13:28:42.134640886Z\",\n      \"endTime\": \"2026-08-12T13:47:44.057070423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001590160\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-pctqk9m4626n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e8d822d-2e8a-489a-bdca-1ae43bb9700c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e8d822d-2e8a-489a-bdca-1ae43bb9700c\",\n      \"name\": \"1e8d822d-2e8a-489a-bdca-1ae43bb9700c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T13:28:42.134640886Z\",\n      \"endTime\": \"2026-08-12T13:47:44.057070423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"4f9682d70a4554dcc68dedb6b65516cb960ece59252528119ea32475b66a8af9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"605ec04d25c79332bd56db79db2496ff2bb7a4eb721fe659ca89ef1def6ac1f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"6cc34f9a635aff56cf6032ad250c9634626556d437adf67fc1b5b781b4924cb9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[1,0,0,0,0,0,3],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-06T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0024100c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5n5w876vz4wl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/e22d396b-a288-47f4-bdf1-eb464612dae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/e22d396b-a288-47f4-bdf1-eb464612dae2\",\n      \"name\": \"e22d396b-a288-47f4-bdf1-eb464612dae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:04:47.92715172Z\",\n      \"endTime\": \"2026-08-12T19:28:50.307111137Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vua42a1p85gdi8mjoaleo34ajvie still exists (deletion dispatched at 2026-08-12T19:04:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0024100c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5n5w876vz4wl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/e22d396b-a288-47f4-bdf1-eb464612dae2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/e22d396b-a288-47f4-bdf1-eb464612dae2\",\n      \"name\": \"e22d396b-a288-47f4-bdf1-eb464612dae2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:04:47.92715172Z\",\n      \"endTime\": \"2026-08-12T19:28:50.307111137Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vua42a1p85gdi8mjoaleo34ajvie still exists (deletion dispatched at 2026-08-12T19:04:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014be288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n6hrw9h6rzsj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b79aa750-1a94-43f1-8635-b5fe6da29eb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b79aa750-1a94-43f1-8635-b5fe6da29eb2\",\n      \"name\": \"b79aa750-1a94-43f1-8635-b5fe6da29eb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:04:12.966485706Z\",\n      \"endTime\": \"2026-08-12T19:28:20.336089092Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vuc59ai6s4r0kfigrf9c2cmjn98g still exists (deletion dispatched at 2026-08-12T19:04:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014be288\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n6hrw9h6rzsj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b79aa750-1a94-43f1-8635-b5fe6da29eb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b79aa750-1a94-43f1-8635-b5fe6da29eb2\",\n      \"name\": \"b79aa750-1a94-43f1-8635-b5fe6da29eb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:04:12.966485706Z\",\n      \"endTime\": \"2026-08-12T19:28:20.336089092Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vuc59ai6s4r0kfigrf9c2cmjn98g still exists (deletion dispatched at 2026-08-12T19:04:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00028ba10\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-fknxhj9rdx4p\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b701fd0b-2852-402a-bd76-04bb471d34c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b701fd0b-2852-402a-bd76-04bb471d34c8\",\n      \"name\": \"b701fd0b-2852-402a-bd76-04bb471d34c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T18:58:22.33048423Z\",\n      \"endTime\": \"2026-08-12T19:22:30.329176043Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vn1h2pjq44l4bti9s24h2348nmf2 still exists (deletion dispatched at 2026-08-12T18:58:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00028ba10\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-fknxhj9rdx4p\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b701fd0b-2852-402a-bd76-04bb471d34c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b701fd0b-2852-402a-bd76-04bb471d34c8\",\n      \"name\": \"b701fd0b-2852-402a-bd76-04bb471d34c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T18:58:22.33048423Z\",\n      \"endTime\": \"2026-08-12T19:22:30.329176043Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vn1h2pjq44l4bti9s24h2348nmf2 still exists (deletion dispatched at 2026-08-12T18:58:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"00ed05eaf7c319c78b8609bf53aa3a2e7d9262156014ce5fa47f1f36b891e8a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"2cc3af52cf3295cee8b395e7d8c907de77bd107308c76d06fbf1d3756e06e03f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"9d3a1233ef4cb7f7453ba3574e52acbcf3c23429e9aa6bdd76f2d039513f055b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3d8b21ba20484aaa0ba81955928ba0600e83c5e3854c628e01a8d708233935f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00159a258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4wznfrtv65kl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/475268e8-a551-439b-a78d-01a047df04ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/475268e8-a551-439b-a78d-01a047df04ef\",\n      \"name\": \"475268e8-a551-439b-a78d-01a047df04ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:01:37.57585238Z\",\n      \"endTime\": \"2026-08-12T19:25:40.350229215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s57012eqkvq06i5h3mhactof6n7879o still exists (deletion dispatched at 2026-08-12T19:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00159a258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4wznfrtv65kl\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/475268e8-a551-439b-a78d-01a047df04ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/475268e8-a551-439b-a78d-01a047df04ef\",\n      \"name\": \"475268e8-a551-439b-a78d-01a047df04ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:01:37.57585238Z\",\n      \"endTime\": \"2026-08-12T19:25:40.350229215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s57012eqkvq06i5h3mhactof6n7879o still exists (deletion dispatched at 2026-08-12T19:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb41b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jmxrwnwqx8lf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/0aaf1273-191d-46fd-866a-12ea803f6545\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/0aaf1273-191d-46fd-866a-12ea803f6545\",\n      \"name\": \"0aaf1273-191d-46fd-866a-12ea803f6545\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:02:55.120678462Z\",\n      \"endTime\": \"2026-08-12T19:27:00.347070634Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vvfk4ag743ihtos8tqksk4v3lm7u still exists (deletion dispatched at 2026-08-12T19:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb41b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jmxrwnwqx8lf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/0aaf1273-191d-46fd-866a-12ea803f6545\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/0aaf1273-191d-46fd-866a-12ea803f6545\",\n      \"name\": \"0aaf1273-191d-46fd-866a-12ea803f6545\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:02:55.120678462Z\",\n      \"endTime\": \"2026-08-12T19:27:00.347070634Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vvfk4ag743ihtos8tqksk4v3lm7u still exists (deletion dispatched at 2026-08-12T19:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"2ae6ff736759395e04388d3b13c6644364f3329f65297e516f7b5fdb0f4aa101","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"e92cc58205948bcc377e6ebf9a9e16dd6ec99683cfb1ca2c14328d7f325af1e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-m9htcw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: failed waiting for cluster=\"cluster-nightly-5-1-m9htcw\" in resourcegroup=\"rg-nightly-5-1-pnw4l5sfdbqd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":34,"prior_last_seen_at":"2026-08-09T23:00:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-pnw4l5sfdbqd/cluster-nightly-5-1-m9htcw should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00179e3f0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-m9htcw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: failed waiting for cluster=\"cluster-nightly-5-1-m9htcw\" in resourcegroup=\"rg-nightly-5-1-pnw4l5sfdbqd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-pnw4l5sfdbqd/cluster-nightly-5-1-m9htcw should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00179e3f0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-m9htcw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: failed waiting for cluster=\"cluster-nightly-5-1-m9htcw\" in resourcegroup=\"rg-nightly-5-1-pnw4l5sfdbqd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-m9htcw in resource group rg-nightly-5-1-pnw4l5sfdbqd, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0021a42d0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l4nzmsf79xg8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-l4nzmsf79xg8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l4nzmsf79xg8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0021a42d0\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l4nzmsf79xg8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-l4nzmsf79xg8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-l4nzmsf79xg8, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2087675652110028800","occurred_at":"2026-08-12T23:00:37Z","signature_id":"81241249699542ae1bd42e545e297f1c7f9120b34640dc9c8d4ee16e94815c21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2087675652110028800","occurred_at":"2026-08-12T23:00:37Z","signature_id":"c6befb4cf0907ebbd2a34bffced134a031018464da512a6b245892b6ed95cdf1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-06T12:27:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-rlrthwb75z45/cluster-candidate-4-22-cvgkbb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088960\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-cvgkbb: failed waiting for cluster=\"cluster-candidate-4-22-cvgkbb\" in resourcegroup=\"rg-candidate-4-22-rlrthwb75z45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/8b6c6c11-4910-43eb-bf9a-96d41817ba00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/8b6c6c11-4910-43eb-bf9a-96d41817ba00\",\n      \"name\": \"8b6c6c11-4910-43eb-bf9a-96d41817ba00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:02:32.298043518Z\",\n      \"endTime\": \"2026-08-12T14:21:35.184163414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-rlrthwb75z45/cluster-candidate-4-22-cvgkbb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088960\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-cvgkbb: failed waiting for cluster=\"cluster-candidate-4-22-cvgkbb\" in resourcegroup=\"rg-candidate-4-22-rlrthwb75z45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/8b6c6c11-4910-43eb-bf9a-96d41817ba00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/8b6c6c11-4910-43eb-bf9a-96d41817ba00\",\n      \"name\": \"8b6c6c11-4910-43eb-bf9a-96d41817ba00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T14:02:32.298043518Z\",\n      \"endTime\": \"2026-08-12T14:21:35.184163414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527122099769344","occurred_at":"2026-08-12T13:10:18Z","signature_id":"dcfa82900b2a4d568a2bc74bf2f5b5b9f0d2dc52d2271d77ba27d902c199153f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e57b2b8e182a0075970014ed4ba7b451352c6477b5e5228e3f75c9584869e231","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-j88vpm5zwmjz/cluster-candidate-4-20-hqcnkd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f2440\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-hqcnkd: failed waiting for cluster=\"cluster-candidate-4-20-hqcnkd\" in resourcegroup=\"rg-candidate-4-20-j88vpm5zwmjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2abaa67f-ecf3-4abe-a630-3de8db310b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2abaa67f-ecf3-4abe-a630-3de8db310b64\",\n      \"name\": \"2abaa67f-ecf3-4abe-a630-3de8db310b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:48:17.939534736Z\",\n      \"endTime\": \"2026-08-13T01:07:24.896522074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-j88vpm5zwmjz/cluster-candidate-4-20-hqcnkd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013f2440\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-hqcnkd: failed waiting for cluster=\"cluster-candidate-4-20-hqcnkd\" in resourcegroup=\"rg-candidate-4-20-j88vpm5zwmjz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2abaa67f-ecf3-4abe-a630-3de8db310b64\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/2abaa67f-ecf3-4abe-a630-3de8db310b64\",\n      \"name\": \"2abaa67f-ecf3-4abe-a630-3de8db310b64\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-13T00:48:17.939534736Z\",\n      \"endTime\": \"2026-08-13T01:07:24.896522074Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087690481667411968","occurred_at":"2026-08-12T23:59:32Z","signature_id":"dc957dcad5897625c97cceb603166ce6cc470c726a13ab52001c8524d7aeda7d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e60588\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gwf29mt4vndw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b84da322-126b-40db-9c02-8fd08857cbaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b84da322-126b-40db-9c02-8fd08857cbaa\",\n      \"name\": \"b84da322-126b-40db-9c02-8fd08857cbaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:06:08.352017084Z\",\n      \"endTime\": \"2026-08-12T19:30:10.499596804Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vuelh16q2upr7d3l759rrt7ivm72 still exists (deletion dispatched at 2026-08-12T19:06:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e60588\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gwf29mt4vndw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b84da322-126b-40db-9c02-8fd08857cbaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b84da322-126b-40db-9c02-8fd08857cbaa\",\n      \"name\": \"b84da322-126b-40db-9c02-8fd08857cbaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-12T19:06:08.352017084Z\",\n      \"endTime\": \"2026-08-12T19:30:10.499596804Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s56vuelh16q2upr7d3l759rrt7ivm72 still exists (deletion dispatched at 2026-08-12T19:06:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087608906141405184","occurred_at":"2026-08-12T18:35:18Z","signature_id":"bde59a0cf1042d1621c2968a5b03c2bb9fee69e758d66b8057cc22720d6b4905","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-06..2026-08-12 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-13","2026-07-20","2026-07-27","2026-08-03"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-05T12:11:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016102a0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jj7mm.apps.aro.cilium-cl.x1ke.centralindia.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016102a0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jj7mm.apps.aro.cilium-cl.x1ke.centralindia.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087527518214033408","occurred_at":"2026-08-12T13:11:59Z","signature_id":"0237e7d6a391afaaa1da230f25139a019cc9e140b2db1f6953c9420e0b3b34d7","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}