{"meta":{"start_date":"2026-07-29","end_date":"2026-07-29","start_at":"2026-07-29T00:00:00Z","end_at":"2026-07-30T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:56:05Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":15,"failed_runs":6,"raw_occurrences":17,"matched_occurrences":17,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":13.333333333333334,"anchor_occurrences":8,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,3,8],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e44d60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8t8h2tgjbrcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"name\": \"e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.496403767Z\",\n      \"endTime\": \"2026-07-29T05:33:46.625119368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e44d60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8t8h2tgjbrcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"name\": \"e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.496403767Z\",\n      \"endTime\": \"2026-07-29T05:33:46.625119368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a2180\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jp5gqq6r46ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"name\": \"f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.662472487Z\",\n      \"endTime\": \"2026-07-29T05:33:46.618172895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a2180\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jp5gqq6r46ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"name\": \"f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.662472487Z\",\n      \"endTime\": \"2026-07-29T05:33:46.618172895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001456540\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ghbtv2frk8jv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"name\": \"07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.670658764Z\",\n      \"endTime\": \"2026-07-29T05:33:46.645534989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001456540\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ghbtv2frk8jv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"name\": \"07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.670658764Z\",\n      \"endTime\": \"2026-07-29T05:33:46.645534989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"3320b7b421abc7495f3d38d4c54d92df2ee5569a5b7507340497c93c97a16b2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"83a466dabf4df209fbbc153c92389d7b33d53abfa5d0ea52538017c262aa8d7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"89ec90a05854a96e4314d56d7b2103aca5da8c5827d4d0912eba0a8b49920eb7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"985863ee16558d554939b5436eded2bc5d8e28deece7209755185db265ecdbd2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"bf6b529a259dfb87e6ef3b60ae798f6e0f770f84da51d8163a2c8622149afbd5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"c46ba0cb07eeb66c683f5e69d56a31d8f06a3223bda86d64c6cbafa4257b8b22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"eee34cb496c5a8c7be29d16abaf5d0a666a957636e08b1c93061315751f8b52f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082285143296315392","occurred_at":"2026-07-29T02:00:39Z","signature_id":"fe1ab3d15d93157c8f567e9b49bfe94cb77c39e9d6c00f3889916cd9f2f74c0b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":13.333333333333334,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-25T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00130a258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00130a258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a2c0a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a2c0a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e48150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e48150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"7e2a79de725776165b8aa00972dd0a55650a63cb3758764c653899700a45bcc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"bc1dfc2c55c0166aaf97f354cd93cc3c56076c1026d4eee5e529342b2fbe5534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"dbef0570edb7869dcc70fa2607e08962e0ef53d745127ecc46f504d82360bd8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082259111117852672","occurred_at":"2026-07-29T00:17:13Z","signature_id":"55c6bf9689c47e7e895ffff2e4c5a16c064dfdccbb11b42f90d9359f0f01c1f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e722c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e722c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","signature_id":"3d8c987c527570b02284ae5b1dd3553b6cc602b543deeb4c7abc806b1b8eb8c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","signature_id":"a6b013ff1d8bdd34482eff2d5f6a091650cd8c4a6b2f95151702b83e709e2576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-20"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-23T07:09:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002462078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gv7f9fvg24h7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"name\": \"1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T11:50:37.856046587Z\",\n      \"endTime\": \"2026-07-29T12:09:40.769017857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002462078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gv7f9fvg24h7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"name\": \"1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T11:50:37.856046587Z\",\n      \"endTime\": \"2026-07-29T12:09:40.769017857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082426114898137088","occurred_at":"2026-07-29T11:20:50Z","signature_id":"fdc5d2be0d25f697e55b562829775e172533b19f5057a9135b273313e4c97e0b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-19T17:04:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","signature_id":"a17e2efd2f02a984b507dcc724ffb9c4f854ad0a7bf6c1ddd6785c1d3ad1414a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":6.666666666666667,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[2,0,0,0,0,1,1],"trend_range":"2026-07-23..2026-07-29 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-20"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-23T12:25:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}