{"meta":{"start_date":"2026-07-20","end_date":"2026-07-26","start_at":"2026-07-20T00:00:00Z","end_at":"2026-07-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T17:51:28Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":22,"failed_runs":10,"raw_occurrences":27,"matched_occurrences":27,"runs_affected":10},"rows":[{"environment":"stg","failure_pattern_id":"4e28048f9d678938ef2ef3a0d4c21d5531d61f9cd55e0237f7ee243424a10e83","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":13.636363636363637,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,0,1,11,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045ef40\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-btrlf49mx4mr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"name\": \"5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:29:57.199203941Z\",\n      \"endTime\": \"2026-07-23T15:49:11.470560955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00045ef40\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-btrlf49mx4mr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"name\": \"5f8bb69e-3dbc-46bc-b77f-ac4c9b9f8f96\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:29:57.199203941Z\",\n      \"endTime\": \"2026-07-23T15:49:11.470560955Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-kr2pqb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02700\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kr2pqb: failed waiting for cluster=\"cluster-zstream-4-20-kr2pqb\" in resourcegroup=\"rg-zstream-upgrade-4-20-5gp2mtdbkppf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"name\": \"a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:33:05.814258572Z\",\n      \"endTime\": \"2026-07-23T15:52:11.319531705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-kr2pqb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e02700\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kr2pqb: failed waiting for cluster=\"cluster-zstream-4-20-kr2pqb\" in resourcegroup=\"rg-zstream-upgrade-4-20-5gp2mtdbkppf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"name\": \"a1b92cfe-14c4-4841-a691-dcb15fd4bada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:33:05.814258572Z\",\n      \"endTime\": \"2026-07-23T15:52:11.319531705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5d0a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kzmh2hsvhvk8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"name\": \"e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:38:51.52148988Z\",\n      \"endTime\": \"2026-07-23T15:57:51.229369914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5d0a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-kzmh2hsvhvk8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"name\": \"e6475fb8-7d66-4a3f-9a11-c8ce9a4ff903\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T15:38:51.52148988Z\",\n      \"endTime\": \"2026-07-23T15:57:51.229369914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"0909626138a55ab7c8dbd794c8b3a77fafa7aac5f3f2c44edb62238255966d8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"2fc2833dd7943e0267fbd875c3cb47629186c64ce8a679a0b0e9dc7f1789a8d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"4c0750c76c5e4a46d72be559656e9da35418529ee0242ad589653e7f2910b0cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"5a371337c5131b79943f94870760e02310f49052df1b8843b9661a8c70afd761","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"6b64c0497cb6a7de0f6df1316cd7aea1515100901d5aeae62f2e291545f5df32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"77259c9c04a7ea64f63b5aa40837444b03f3056038b0d685ff4e584482f0e763","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"bb17f5b2b19e58bc0ec041403d55521015c5e5fb9b4d2b13c8c0c5fe246700a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"ca2a57c00f153b6d7488071e8d97ffb596848ffc1e258c95a04400b39f327f42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"f1aebeeb7492bd08c71a6b12c63343c08386503b6840ef053c0c3d9ecad34f5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"fe4559ffec4f3ebb39829cc2d25299d572c52b91485aafaa31fc6c7b5d86ebf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080143850206138368","occurred_at":"2026-07-23T04:11:53Z","signature_id":"ff776f26d065fc5418b757e9c6b4187c71a02c6c48dc47fb83ce32ca7f8ef27a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2079941546198700032","occurred_at":"2026-07-22T14:48:01Z","signature_id":"15954d7592a9121f1bbb5491dbcb008720f777e3e6a9a5ba79923d0d07309094","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-name-t94r677w7g5j\", caused by: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":9.090909090909092,"anchor_occurrences":3,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,0,1,2],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-06","2026-07-13"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-13T02:00:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fa0198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-name-t94r677w7g5j\", caused by: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fa0198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"patch-name-t94r677w7g5j\", caused by: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded, error: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t94r677w7g5j\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster patch-name-cluster in resource group patch-name-t94r677w7g5j, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028f23a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"private-keyvault-86b8lc297hk6\", caused by: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-86b8lc297hk6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-86b8lc297hk6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-86b8lc297hk6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-86b8lc297hk6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0028f23a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"private-keyvault-86b8lc297hk6\", caused by: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-86b8lc297hk6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-86b8lc297hk6, error: context deadline exceeded, error: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-86b8lc297hk6\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster private-kv-cluster in resource group private-keyvault-86b8lc297hk6, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00123e2b8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-z4fpldkzrbd9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-z4fpldkzrbd9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00123e2b8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-z4fpldkzrbd9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-z4fpldkzrbd9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-z4fpldkzrbd9, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2081198432965365760","occurred_at":"2026-07-26T02:00:53Z","signature_id":"0ab01ed82376507574059677dba5c30fa0ce245eb2474028194536aa13f1db4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2081198432965365760","occurred_at":"2026-07-26T02:00:53Z","signature_id":"74f649e0a96d40b547231fa0b2430e4d29938e3268528adfa67cdd73c0d75d34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2081152736228806656","occurred_at":"2026-07-25T23:00:52Z","signature_id":"c06c31c7f2e634e557e15eba8ac32eb66c92c5d49c2457984db7a6b0a67a1fef","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":9.090909090909092,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,2,0,0,0,1,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-18T23:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:138]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ef8c0\u003e: \n    failed to create NodePool np-2: failed waiting for nodepool=\"np-2\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7pjp2fntgwhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"name\": \"c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-24T02:16:26.672204644Z\",\n      \"endTime\": \"2026-07-24T02:35:27.643263493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:138]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ef8c0\u003e: \n    failed to create NodePool np-2: failed waiting for nodepool=\"np-2\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7pjp2fntgwhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"name\": \"c569a933-5592-4cb9-951a-4e439ed8e930\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-24T02:16:26.672204644Z\",\n      \"endTime\": \"2026-07-24T02:35:27.643263493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:95]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c774c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bkhsht\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f057642-f8a4-4c52-8a2f-292b71945df2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f057642-f8a4-4c52-8a2f-292b71945df2\",\n      \"name\": \"2f057642-f8a4-4c52-8a2f-292b71945df2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:13:02.800607117Z\",\n      \"endTime\": \"2026-07-20T23:32:21.039712476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:95]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c774c0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bkhsht\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f057642-f8a4-4c52-8a2f-292b71945df2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f057642-f8a4-4c52-8a2f-292b71945df2\",\n      \"name\": \"2f057642-f8a4-4c52-8a2f-292b71945df2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:13:02.800607117Z\",\n      \"endTime\": \"2026-07-20T23:32:21.039712476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001076048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9m888\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88dd770-5841-4db8-a438-9fa916137ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88dd770-5841-4db8-a438-9fa916137ff8\",\n      \"name\": \"b88dd770-5841-4db8-a438-9fa916137ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:19:14.772122124Z\",\n      \"endTime\": \"2026-07-20T23:38:20.899291077Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001076048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9m888\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88dd770-5841-4db8-a438-9fa916137ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b88dd770-5841-4db8-a438-9fa916137ff8\",\n      \"name\": \"b88dd770-5841-4db8-a438-9fa916137ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:19:14.772122124Z\",\n      \"endTime\": \"2026-07-20T23:38:20.899291077Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080473235987632128","occurred_at":"2026-07-24T02:00:47Z","signature_id":"d9d95b5e04d2638d898684563003edb95885592f2ef99fddfee432bdb213cde3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2079340761865785344","occurred_at":"2026-07-20T23:00:44Z","signature_id":"d10a04490bcb5c2180ffbde28b777b9744e5033fcd0a0e9c57e5a1f619000eba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2079340761865785344","occurred_at":"2026-07-20T23:00:44Z","signature_id":"d4d44ef083d7f699ac7620446fea98b3718b0bbd750e4b93f44e48a7948bd9da","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,0,1,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-bxb82trtk28r/cluster-nightly-4-23-f626bt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9c1a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-f626bt: failed waiting for cluster=\"cluster-nightly-4-23-f626bt\" in resourcegroup=\"rg-nightly-4-23-bxb82trtk28r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\",\n      \"name\": \"0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-25T23:02:57.404100166Z\",\n      \"endTime\": \"2026-07-25T23:22:19.985017273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-bxb82trtk28r/cluster-nightly-4-23-f626bt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9c1a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-f626bt: failed waiting for cluster=\"cluster-nightly-4-23-f626bt\" in resourcegroup=\"rg-nightly-4-23-bxb82trtk28r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\",\n      \"name\": \"0f19b0d2-34e7-42c9-99c1-fa33aeb87a0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-25T23:02:57.404100166Z\",\n      \"endTime\": \"2026-07-25T23:22:19.985017273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-24nfpmj67m6s/cluster-nightly-4-23-cjrw8r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013903c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-cjrw8r: failed waiting for cluster=\"cluster-nightly-4-23-cjrw8r\" in resourcegroup=\"rg-nightly-4-23-24nfpmj67m6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b94d127-cbd8-4a47-8b57-4bcae269b1a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b94d127-cbd8-4a47-8b57-4bcae269b1a0\",\n      \"name\": \"1b94d127-cbd8-4a47-8b57-4bcae269b1a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:39.002058547Z\",\n      \"endTime\": \"2026-07-21T23:24:56.606069748Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-23-24nfpmj67m6s/cluster-nightly-4-23-cjrw8r should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013903c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-cjrw8r: failed waiting for cluster=\"cluster-nightly-4-23-cjrw8r\" in resourcegroup=\"rg-nightly-4-23-24nfpmj67m6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b94d127-cbd8-4a47-8b57-4bcae269b1a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b94d127-cbd8-4a47-8b57-4bcae269b1a0\",\n      \"name\": \"1b94d127-cbd8-4a47-8b57-4bcae269b1a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:39.002058547Z\",\n      \"endTime\": \"2026-07-21T23:24:56.606069748Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2081152736228806656","occurred_at":"2026-07-25T23:00:52Z","signature_id":"e19fcc29d6190114f70d31e9b6a1cefc4232179087c41185c63f0859ac2b4b2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2079703154030546944","occurred_at":"2026-07-21T23:00:45Z","signature_id":"931d2c22bf249c2f7357aca6b7a43b50fbb907d7e800ef1fe05bc24b3349a5b7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-pq7q77, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: failed waiting for cluster=\"cluster-nightly-5-0-pq7q77\" in resourcegroup=\"rg-nightly-5-0-6qqhqp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":52,"prior_last_seen_at":"2026-07-19T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f326f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qp8jkwnz8g6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qp8jkwnz8g6c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f326f0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qp8jkwnz8g6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qp8jkwnz8g6c, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-6qqhqp/cluster-nightly-5-0-pq7q77 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001608540\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-pq7q77, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: failed waiting for cluster=\"cluster-nightly-5-0-pq7q77\" in resourcegroup=\"rg-nightly-5-0-6qqhqp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-6qqhqp/cluster-nightly-5-0-pq7q77 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001608540\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-pq7q77, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: failed waiting for cluster=\"cluster-nightly-5-0-pq7q77\" in resourcegroup=\"rg-nightly-5-0-6qqhqp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-pq7q77 in resource group rg-nightly-5-0-6qqhqp, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"fc915ba250e390d158815e863f80c25f3e4cbc8021425841dcebdf7862c3f2e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2079340761865785344","occurred_at":"2026-07-20T23:00:44Z","signature_id":"a6c4028ca98f7000864fed551d632ffd561433d1588c696d94feceb7c3719aa5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"43b4c0ecc7cfb52979675ed0f02de9aacdd82a8bb5bb24f9b95172e131f6846b","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-2hkmp8kgmll4/cluster-candidate-4-22-zqcp95 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zqcp95: failed waiting for cluster=\"cluster-candidate-4-22-zqcp95\" in resourcegroup=\"rg-candidate-4-22-2hkmp8kgmll4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"name\": \"7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:57:03.082121392Z\",\n      \"endTime\": \"2026-07-23T15:16:10.601844479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-22-2hkmp8kgmll4/cluster-candidate-4-22-zqcp95 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zqcp95: failed waiting for cluster=\"cluster-candidate-4-22-zqcp95\" in resourcegroup=\"rg-candidate-4-22-2hkmp8kgmll4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"name\": \"7dd0acb6-1bcd-4cbd-a77e-1e2698a80ca7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T14:57:03.082121392Z\",\n      \"endTime\": \"2026-07-23T15:16:10.601844479Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"8a2bdaa74464be4ae4f7085471004a2892b6b516012862043fe6c37bb6ddae92","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080300458475261952","occurred_at":"2026-07-23T14:34:14Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"2c1b6e2e9a0cffd0b5c8baae2a1873f1cdcec411d83a27ab1b9a7bc7d25a1dc2","failure_pattern":"failed to create cluster role binding for external auth \u003cexternal-auth\u003e VM","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:328]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:328]: failed to create cluster role binding for external auth via VM\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080473235987632128","occurred_at":"2026-07-24T02:00:47Z","signature_id":"4f4c331b7bc2053d2469b256d4be2c082415cc0b7c4a8902668003f5d043798f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[1,0,0,0,0,1,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-22","2026-07-06","2026-07-13"],"prior_runs_affected":9,"prior_last_seen_at":"2026-07-19T02:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-9p9q96\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e200\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5s5tl.apps.aro.t4h5n3m8h9b6b1y.pmy9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-9p9q96\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e200\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-5s5tl.apps.aro.t4h5n3m8h9b6b1y.pmy9.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2080621562049859584","occurred_at":"2026-07-24T11:50:11Z","signature_id":"d4ce604404a11bb526e6130361b3d54203e84830c3cb947360b22e2e753d01c1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":2,"prior_last_seen_at":"2026-07-13T23:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:234]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:234]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2080835644103331840","occurred_at":"2026-07-25T02:00:52Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}