{"meta":{"date":"2026-09-23","timezone":"UTC","generated_at":"2026-10-14T20:12:14Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":97,"failed_runs":69,"runs_with_occurrences":69,"runs_with_failure_pattern_matches":64,"runs_with_unmatched_occurrences":5,"runs_without_occurrence_rows":28,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"65d675bb300fd87c5f0a14ac91d2d6e076693fe8","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T23:42:18Z","started_at":"2026-09-23T23:42:18Z","completed_at":"2026-09-24T01:52:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:04:21.077156838Z","timing_metadata_checked_at":"2026-09-24T02:04:21.077156838Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:04:21.080905225Z","region_metadata_checked_at":"2026-09-24T02:04:21.080905225Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1ea7807c6150bfdc24425215651cf3c674289f85fa8a1bf852a4b35a4da92a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","occurred_at":"2026-09-23T23:42:18Z","failed_at":"e2e","signature_id":"9d20053e21637cc884e3f5b506bcfcb59a5c2cd22b2f8ee5ff24cb97bfeb11c4","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"5ed56c9c71438fc12bb60f11851724f51fb25982","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T23:26:51Z","started_at":"2026-09-23T23:26:51Z","completed_at":"2026-09-24T01:23:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T01:33:52.001171167Z","timing_metadata_checked_at":"2026-09-24T01:33:52.001171167Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T01:33:52.004913354Z","region_metadata_checked_at":"2026-09-24T01:33:52.004913354Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6f068d648acbe97065cddd0536480d056836c090b7aec872bc2337eeba3a1451","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","occurred_at":"2026-09-23T23:26:51Z","failed_at":"e2e","signature_id":"c8ce66f37f9d13dc657060400db223a90301ac4438226c58a0c01cac44d1ec13","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:58:15Z","started_at":"2026-09-23T22:58:15Z","completed_at":"2026-09-23T23:32:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:37:40.107438549Z","timing_metadata_checked_at":"2026-09-23T23:37:40.107438549Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:37:40.111155837Z","region_metadata_checked_at":"2026-09-23T23:37:40.111155837Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ef1d48548595682424e0987bbda06126314c2cddc3e79f2dc6774c84dc69724b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","occurred_at":"2026-09-23T22:58:15Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T23:25:15.442Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T23:25:17.399Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T23:25:18.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9983616%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe\ntime=2026-09-23T23:27:49.191Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9983616/providers/Microsoft.Resources/deployments/72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe/operationStatuses/08584114009676534307\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T22:12:26Z","started_at":"2026-09-23T22:12:26Z","completed_at":"2026-09-24T00:28:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:40:41.327495438Z","timing_metadata_checked_at":"2026-09-24T00:40:41.327495438Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:40:41.330671228Z","region_metadata_checked_at":"2026-09-24T00:40:41.330671228Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"430f010f6d1f2a44b7240892451e543779c2a61277f763e28c6efce15a5f460f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"alert","signature_id":"125273ad41e8d32bfd551d1beb0bc45baf8569efc2fb05f6f2496e0c967d64ac","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T23:16:20Z\n  Ended:       2026-09-23T23:41:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7507200-svc\", component=\"backend\", name=\"datadump\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"e2cb31fa4f45c3e266ee146be5d7f61e1db49c5f03696eb476670c8afd640f35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"e2e","signature_id":"0c93d7531b51f1ea100516ccd01b0a1c6615ecf8a2bbdc599e9f78e9e71a81e6","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7cb5e18e18a5375243672eb6ca5a8a28d188dcd703c199519379851de29ef9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"alert","signature_id":"45c5b936de9aef65cab348b5a9c37cd387b152250ed997df53a6e8778dabfb08","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T00:08:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7507200-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rpfwd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rpfwd (namespace arobit) on cluster ci01-j7507200-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"45619559b0273524877730f5066212561774a32d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:03:16Z","started_at":"2026-09-23T22:03:16Z","completed_at":"2026-09-24T00:16:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:30:24.492408695Z","timing_metadata_checked_at":"2026-09-24T00:30:24.492408695Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:30:24.496705081Z","region_metadata_checked_at":"2026-09-24T00:30:24.496705081Z"},"failed_at":["alert","e2e"],"failed_test_count":15,"occurrences":[{"row_id":"076deb2bb8db23bd54067a1e7d49c41108f6e58cb080277d2db57ffcf850f19a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"0c65583f9cf2766f075edef8345e43c644c0d3a9639fd26af02b95e481bc76ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n      \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n      \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n      \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n      \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23751d3251cb875ef3907a1d20bc37a85b9f161ea167d6599a7a41a94aed159d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"266bf773c9ea2839b61a117c9276cf7872810cf8b981f8d2081dee6bd5dac94e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"ef425b9fc641abb8b932709dc29ce158807de6ae50f47a33b3db89e10d6a6b93","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362d00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvfnk745g57p\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\",\n      \"name\": \"277377a5-4429-4698-a365-74f878b05216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:00:00.50151725Z\",\n      \"endTime\": \"2026-09-23T23:29:02.685864461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362d00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvfnk745g57p\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\",\n      \"name\": \"277377a5-4429-4698-a365-74f878b05216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:00:00.50151725Z\",\n      \"endTime\": \"2026-09-23T23:29:02.685864461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"312344568f7eabc86a986b50d033259549f76ac9ce1669463a8e2a7b7e4cb724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"a08dcc436350e9ea389eb3d5efb0662df729d203d30b15f5920616ae5fc224ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-q7fxr6l2s4gv/cluster-candidate-4-23-8fnk5c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f92780\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-8fnk5c: failed waiting for cluster=\"cluster-candidate-4-23-8fnk5c\" in resourcegroup=\"rg-candidate-4-23-q7fxr6l2s4gv\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"name\": \"f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:55:37.049841827Z\",\n      \"endTime\": \"2026-09-23T23:24:43.011652216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-q7fxr6l2s4gv/cluster-candidate-4-23-8fnk5c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f92780\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-8fnk5c: failed waiting for cluster=\"cluster-candidate-4-23-8fnk5c\" in resourcegroup=\"rg-candidate-4-23-q7fxr6l2s4gv\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"name\": \"f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:55:37.049841827Z\",\n      \"endTime\": \"2026-09-23T23:24:43.011652216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3dbc36fcf4671bf9b06d29711c707f4fe9adba325f6772fd56d1984b41843889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"d8dcd8028ce1adacc03f46fb0158f289f63ae31f72e8e4ce40b790b255cba87a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696ea0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cbz2zvj9pgcc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"name\": \"8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:17.979672195Z\",\n      \"endTime\": \"2026-09-23T23:51:22.771798672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696ea0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cbz2zvj9pgcc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"name\": \"8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:17.979672195Z\",\n      \"endTime\": \"2026-09-23T23:51:22.771798672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"437b11c1003b57bc8c736bd38e421f561b1e74ecf24940ebcf6cbbc75c7092f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"7f9ed0d0bd04a4b470fe9147f2b70a32e6b959bfcc09d740b923b169f280b98f","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:23:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:23:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T23:31:17Z\n  Ended:       2026-09-23T23:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T23:32:18Z\n  Ended:       2026-09-23T23:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5e36a55b276663958b0731618b72cebe5b692e1584f517184003a78f0448bb5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"0a6194beb78b8c406f21d0c39e8f4938c36409cacbef3176434d6c29c859c82b","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000854de0\u003e: \n    failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n      \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n      \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000854de0\u003e: \n    failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n      \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n      \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88276530cb10d38731d3e2465e68aab34992770bb41aa55a68b8fd83516b10e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"cb11f1de63b86677e85bb10ffef4875807561668b3123fc194016353354c7ac8","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001006fc0\u003e: \n    failed to create HCP cluster np-dg-minor-4p9zqn: failed waiting for cluster=\"np-dg-minor-4p9zqn\" in resourcegroup=\"rg-np-dg-minor-4p9zqn-9pzvmjjxxgwg\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"name\": \"95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:20:11.073191713Z\",\n      \"endTime\": \"2026-09-23T23:49:12.786080022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001006fc0\u003e: \n    failed to create HCP cluster np-dg-minor-4p9zqn: failed waiting for cluster=\"np-dg-minor-4p9zqn\" in resourcegroup=\"rg-np-dg-minor-4p9zqn-9pzvmjjxxgwg\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"name\": \"95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:20:11.073191713Z\",\n      \"endTime\": \"2026-09-23T23:49:12.786080022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89319e042db26af717577bff4e522fc1770f719760147b3b4aef8d31ff17b598","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"813c3b4a34551005c00d10be781d74a14c7bbb31e37f764d60afab74ae06652e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000464ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8l6vvszb7492\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"name\": \"53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:58:01.487898788Z\",\n      \"endTime\": \"2026-09-23T23:27:02.7845863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000464ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8l6vvszb7492\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"name\": \"53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:58:01.487898788Z\",\n      \"endTime\": \"2026-09-23T23:27:02.7845863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9bd4609fe496c815f9273292b947da68629a4f405a8604329abb7850cf226c4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"c525979fb2d08507f3d446b04e74f3e5b602e0906716c705201cd81002770ba7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009068a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fx77xd4jlzgc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\",\n      \"name\": \"62671277-0174-4102-8a19-fd81304df39c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:18.488865764Z\",\n      \"endTime\": \"2026-09-23T23:25:22.804222005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009068a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fx77xd4jlzgc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\",\n      \"name\": \"62671277-0174-4102-8a19-fd81304df39c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:18.488865764Z\",\n      \"endTime\": \"2026-09-23T23:25:22.804222005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa806b49bd04b18ac8b36f6cc89c4dfd0b51b1f4bd93031189185d825893e020","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"4342f7d49a31fc62a7cd4188188d768f7c7622da6926b4fcd1d52c2f7741506a","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n      \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n      \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ab91de0f007b40686b47998f501a28ba6e277273eeed911c6aa81abb5dac636e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"7fc420454baa557ad64e773af2ec483367dc179f35828ca014109dbc19bd2053","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372520\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ff8h8wnjvp8f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"name\": \"2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.324238915Z\",\n      \"endTime\": \"2026-09-23T23:25:22.950789091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372520\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ff8h8wnjvp8f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"name\": \"2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.324238915Z\",\n      \"endTime\": \"2026-09-23T23:25:22.950789091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1f15ab51d9208634ab1b7d34df111203f59d19789570a458e7b666a7229f66f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"270a9593b205f1bd6398600a798ec9e0946dfbdebb1375f7feaf55d764c6e286","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:58:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-pbxrh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-pbxrh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"ee3001369776d27b42d98f7738278bc7701ccf7902adeef5f375b5ec9330fa7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"9eb53b927bd56443c24b7b20ca5bab6adea62beccaa9c3e083e3ba49ff15f1b7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ce480\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xcpdnghpn9qj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"name\": \"4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:18.709940835Z\",\n      \"endTime\": \"2026-09-23T23:51:22.784957829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ce480\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xcpdnghpn9qj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"name\": \"4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:18.709940835Z\",\n      \"endTime\": \"2026-09-23T23:51:22.784957829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3e55e1f30598c07bc5a0984f17970517152e1fcd91e5cbdb72e9ed50b2242f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"fbe2fd343d3a9197d314d9b85e39c50bd319b22385b4b05cd0ef79ae96f47f43","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:33:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:01:19Z","started_at":"2026-09-23T22:01:19Z","completed_at":"2026-09-23T22:38:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:43:43.432811327Z","timing_metadata_checked_at":"2026-09-23T22:43:43.432811327Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:43:43.435766917Z","region_metadata_checked_at":"2026-09-23T22:43:43.435766917Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d6385eedc75f05f9b9b7045b22f9fce4034aa5bb3f5d5a31ef5e64ae1c76cc6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","occurred_at":"2026-09-23T22:01:19Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T22:31:07.038Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T22:31:08.814Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T22:31:10.117Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8206336%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1\ntime=2026-09-23T22:34:11.270Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8206336/providers/Microsoft.Resources/deployments/cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1/operationStatuses/08584114042161978286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"3eb10bd2f1feb2c6d62a69754d5b84aa95d9f23b","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T21:27:07Z","started_at":"2026-09-23T21:27:07Z","completed_at":"2026-09-23T23:43:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:59:25.843401859Z","timing_metadata_checked_at":"2026-09-23T23:59:25.843401859Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:59:25.84600615Z","region_metadata_checked_at":"2026-09-23T23:59:25.84600615Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0e077d24ef922eefe9593cbfe5ef73d6b62a666d92679680958e6cb67073bbf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","occurred_at":"2026-09-23T21:27:07Z","failed_at":"alert","signature_id":"aef4de3c1d8c7e8e67123da5e644cdc4d14dd0dceea99369d4b941edf923b506","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T22:48:48Z\n  Ended:       2026-09-23T23:00:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-869c9489f8-dbnnd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-869c9489f8-dbnnd (namespace kube-applier) on cluster ci01-j1922944-mgmt-1 has averaged 124.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:16:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z6smt\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z6smt (namespace arobit) on cluster ci01-j1922944-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"open","pr_sha":"06b9f9889389f9b6bf8ca6bd99239ae9cb9ee430","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T21:24:41Z","started_at":"2026-09-23T21:24:41Z","completed_at":"2026-09-23T23:43:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:59:25.417151424Z","timing_metadata_checked_at":"2026-09-23T23:59:25.417151424Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:59:25.420744713Z","region_metadata_checked_at":"2026-09-23T23:59:25.420744713Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2c18064313d4aa4d755b3671f909b3d699dcf73ef53aeaefa3d3e31f167ee63a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","occurred_at":"2026-09-23T21:24:41Z","failed_at":"alert","signature_id":"d71eb8d3e94b0946ce9d45150994a79c1da31f720176e1cb83633088dcca59fb","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:11:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8888448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m4cdw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m4cdw (namespace arobit) on cluster ci01-j8888448-svc has averaged 124.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T20:49:32Z","started_at":"2026-09-23T20:49:32Z","completed_at":"2026-09-23T21:15:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:20:50.37086223Z","timing_metadata_checked_at":"2026-09-23T21:20:50.37086223Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:20:50.373413821Z","region_metadata_checked_at":"2026-09-23T21:20:50.373413821Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7a602ff1a6bb548cf367efedf993adb662f529022a3c53aab3005d2e8bb376ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","occurred_at":"2026-09-23T20:49:32Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T21:07:35.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T21:07:36.837Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T21:07:38.387Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4221952%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee\ntime=2026-09-23T21:10:39.591Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4221952/providers/Microsoft.Resources/deployments/5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee/operationStatuses/08584114092280138717\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T20:49:31Z","started_at":"2026-09-23T20:49:31Z","completed_at":"2026-09-23T23:01:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:15:56.108448891Z","timing_metadata_checked_at":"2026-09-23T23:15:56.108448891Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:15:56.11224348Z","region_metadata_checked_at":"2026-09-23T23:15:56.11224348Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f883f79f8b55764234ee4ffc0584490b5c4c9ffe51050e4c7362b3111039a202","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","occurred_at":"2026-09-23T20:49:31Z","failed_at":"alert","signature_id":"a051597c46d9f96e8b20b9310e059e4f799849ba3ec132c05941edb67268ea29","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:47:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8872448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z6lp4\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z6lp4 (namespace arobit) on cluster ci01-j8872448-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102859350493106176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T20:35:13Z","started_at":"2026-09-23T20:35:13Z","completed_at":"2026-09-23T22:54:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:05:25.364997409Z","timing_metadata_checked_at":"2026-09-23T23:05:25.364997409Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:05:25.368402398Z","region_metadata_checked_at":"2026-09-23T23:05:25.368402398Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102859342163218432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T20:35:11Z","started_at":"2026-09-23T20:35:11Z","completed_at":"2026-09-23T23:01:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:15:55.971677911Z","timing_metadata_checked_at":"2026-09-23T23:15:55.971677911Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:15:55.979653287Z","region_metadata_checked_at":"2026-09-23T23:15:55.979653287Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:18:53Z","started_at":"2026-09-23T20:18:53Z","completed_at":"2026-09-23T22:48:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:54:38.834260316Z","timing_metadata_checked_at":"2026-09-23T22:54:38.834260316Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:54:38.837581695Z","region_metadata_checked_at":"2026-09-23T22:54:38.837581695Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0c46088f1706b0c9cef2514ca376e7b7bd4608af99ac1240c4e740789a121804","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","failed_at":"alert","signature_id":"23e9b2995fc9ded2e9caf55c0eceabc998d92c6176e04363cf44c1eee0ec317d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:52:45Z\n  Ended:       2026-09-23T22:01:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4706560-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4706560-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"bbcb0abfca75bcbb8fe751bdb7264682b1d61b5e44430261fcd4cfc9355f1bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","failed_at":"alert","signature_id":"254dd39e2f0e4cec1b71ce202f610ab07b9da883c1b746d73e09e4f13421506a","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:30:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4706560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rsvxn\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rsvxn (namespace arobit) on cluster ci01-j4706560-svc has averaged 123.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6733,"pr_state":"closed","pr_sha":"d68f0ce93df0cac968cceb0b5840311d2eb6670f","final_merged_sha":"7df09e82565752899c9669782b8ac3ab38a07b3f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:14:08Z","started_at":"2026-09-23T20:14:08Z","completed_at":"2026-09-23T20:48:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.457979531Z","timing_metadata_checked_at":"2026-09-23T20:59:01.457979531Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.461297921Z","region_metadata_checked_at":"2026-09-23T20:59:01.461297921Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8e576a23eb4f8515fd21dc9df70986d7ac956a95d6c83f40daa42ac4e1f7f4e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","occurred_at":"2026-09-23T20:14:08Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T20:41:08.730Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T20:41:10.239Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T20:41:11.440Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8827264%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701\ntime=2026-09-23T20:43:42.398Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8827264/providers/Microsoft.Resources/deployments/c45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701/operationStatuses/08584114108148792156\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"closed","pr_sha":"1191c30708aa874c2ef1a55f27fe3330c8bcc8bf","final_merged_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:12:38Z","started_at":"2026-09-23T20:12:38Z","completed_at":"2026-09-23T22:20:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:33:38.090698993Z","timing_metadata_checked_at":"2026-09-23T22:33:38.090698993Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:33:38.094258281Z","region_metadata_checked_at":"2026-09-23T22:33:38.094258281Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2d03499c55e8216bb099d24002bf79e79b6d73eccbd7a3146233f0ddd84530d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","occurred_at":"2026-09-23T20:12:38Z","failed_at":"e2e","signature_id":"5278202e490f8d8822dd976e22de925b821240a96d32e8bb0148f9031ab25a7c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084af40\u003e: \n    DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084af40\u003e: \n    DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7116/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102852545683656704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7116,"pr_state":"closed","pr_sha":"e3930b456cc9b031d9dc722fd04de3be63742679","final_merged_sha":"e3930b456cc9b031d9dc722fd04de3be63742679","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T20:08:10Z","started_at":"2026-09-23T20:08:10Z","completed_at":"2026-09-23T22:29:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:43:43.646169524Z","timing_metadata_checked_at":"2026-09-23T22:43:43.646169524Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:43:43.650650309Z","region_metadata_checked_at":"2026-09-23T22:43:43.650650309Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7110,"pr_state":"closed","pr_sha":"a3adee787cd6da9312ca13a764ff882bb046be85","final_merged_sha":"ce66d7285ab6bfc8f735b0e7c24632e94158eb25","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:01:44Z","started_at":"2026-09-23T20:01:44Z","completed_at":"2026-09-23T21:48:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:02:11.938577107Z","timing_metadata_checked_at":"2026-09-23T22:02:11.938577107Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:02:11.942248894Z","region_metadata_checked_at":"2026-09-23T22:02:11.942248894Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2125c7cacedb8cb4f1d5067e5d374665bb3117595db086980b19e6c1330af98a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"88d7c43f7ba21a8263fb8b4c871ac116be74ea1e4b599156e118374c23dbf6bf","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n      \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n      \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44f09923d562e27a236d36bc1632c33c05c0b0520c5b52985a2e7e01298b8152","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"35009a2d8ec15af6629b3b447ac330d1c5376735efbdae49f03535446218b76a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive"}},{"row_id":"86c9b946b34abd6efbdf38c85270957a16adbe8a55f2a532bb3416a6ccea5cdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"alert","signature_id":"d5d990830365516d1467804cae51c493981b29d7c2296cdf7adf893366ce4712","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:26:02Z\n  Ended:       2026-09-23T21:34:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4857344-svc\", component=\"frontend\", method=\"post\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4857344-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"ff76d3c516feffc1d05d7f131f77afbdb669a31b0b45c409ea0002b62bfc9e8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"05cb3f86a6f99432ca7873642f1755e0d1de74704c579e10519fe31b0a3ab239","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","failure_pattern":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6837/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850353635528704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6837,"pr_state":"closed","pr_sha":"c130c3ebb8cf261cdc06d58875ff85ef5ce110ea","final_merged_sha":"c130c3ebb8cf261cdc06d58875ff85ef5ce110ea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:59:27Z","started_at":"2026-09-23T19:59:27Z","completed_at":"2026-09-23T22:17:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:23:03.069123273Z","timing_metadata_checked_at":"2026-09-23T22:23:03.069123273Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:23:03.073746258Z","region_metadata_checked_at":"2026-09-23T22:23:03.073746258Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:52:03Z","started_at":"2026-09-23T19:52:03Z","completed_at":"2026-09-23T22:12:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:23:03.196964052Z","timing_metadata_checked_at":"2026-09-23T22:23:03.196964052Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:23:03.199663743Z","region_metadata_checked_at":"2026-09-23T22:23:03.199663743Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"288da12b2fc1a731c02d9309ef9dd65fd56c9495312d708b48136a6b87f01ffd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"e2e","signature_id":"527306b3dde4b6e298931d98ad45937c8ba364a99fa4d7c4de2caf95207dcd18","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n      \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n      \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"87069f61b46c6ece6ad6525cf89c29d0b5ef8af791964258d9c83354123d8c5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"9766aea3f661a15c4b1ef2c8ce948e7447e6a77d814fe12d510f6ddd5782ae00","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:01:37Z\n  Ended:       2026-09-23T21:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9340032-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a\", pod=\"router-85d7d5999-qx49z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a/router-85d7d5999-qx49z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e1988dc23cebaccf396951ccb5302059122218cb5fdcdedbe06c4e59cd7b9e0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"9768ce66171b43df472ff44180bc3d8e585fee001b4346d3ab9374574a7b151f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T20:59:04Z\n  Ended:       2026-09-23T21:31:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9340032-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9340032-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e918fac622b4b68875b8c8bef71c125c8cdb9545fbb74a97c14781c61e7e7b4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"71f4c03d6bb4614602efd1761f8bbf73a703154b3346a44390285f109ca0b681","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:21:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:36:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:36:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:21:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:42:53Z","started_at":"2026-09-23T19:42:53Z","completed_at":"2026-09-23T20:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:27:19.077217498Z","timing_metadata_checked_at":"2026-09-23T20:27:19.077217498Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:27:19.080003889Z","region_metadata_checked_at":"2026-09-23T20:27:19.080003889Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9ef4b565b50ca9ed86fe30a58ef283acdfb6647b2e61e4ddb687f766d3a1f1e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","occurred_at":"2026-09-23T19:42:53Z","failed_at":"provision","signature_id":"b83201cdb7e435a0abab8639b944505f58098f612295a321d14698e824fe6511","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T20:05:19.475Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-23T20:10:16.316Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-23T20:05:19.482013132Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.259341341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.679397764Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314281955Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":\\\"Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/deployments/aks-cluster-create/operations?api-version=2025-04-01\\\\\\\": stream error: stream ID 41; CANCEL; received from peer\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620","failure_pattern":"{\"level\":\"ERROR\",\"msg\":\"run failed\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5982976-mgmt-2\",\"clusterName\":\"ci01-j5982976-mgmt-2\",\"region\":\"westus3\",\"err\":\"waiting for deployment","search_query":"{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840328758235136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:19:37Z","started_at":"2026-09-23T19:19:37Z","completed_at":"2026-09-23T21:45:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:51:43.693870682Z","timing_metadata_checked_at":"2026-09-23T21:51:43.693870682Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:51:43.698837566Z","region_metadata_checked_at":"2026-09-23T21:51:43.698837566Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:19:30Z","started_at":"2026-09-23T19:19:30Z","completed_at":"2026-09-23T21:36:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:41:34.401949057Z","timing_metadata_checked_at":"2026-09-23T21:41:34.401949057Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:41:34.404847648Z","region_metadata_checked_at":"2026-09-23T21:41:34.404847648Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"049f8a3afeb95eb2006b04c1470856457c5191c031a088329aade1deaa24270e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","occurred_at":"2026-09-23T19:19:30Z","failed_at":"alert","signature_id":"8d0ffdcc532373f83199e900f86943923bf9464de78da96a0bc81c5d10748030","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T21:21:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5527936-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d7xzw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-d7xzw (namespace arobit) on cluster ci01-j5527936-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T19:19:27Z","started_at":"2026-09-23T19:19:27Z","completed_at":"2026-09-23T21:35:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:41:34.267111899Z","timing_metadata_checked_at":"2026-09-23T21:41:34.267111899Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:41:34.273332779Z","region_metadata_checked_at":"2026-09-23T21:41:34.273332779Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2cda3228746978234f3989339799c220df7461d9ce34fea4f5f7fbc725df6f63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","failed_at":"e2e","signature_id":"3fec0289f4e859b2672b4ecfd1c53de11f801ca25e568cb6fd49601565fabc04","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, dns-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e608ced051e13f5963c01177b86dc7083dc3ec405629cacf72c4d113e7a2627c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","failed_at":"alert","signature_id":"f01e8023db70ac4eb379611e2c78c4d54f1aeadca78f25f3826d0ffec54edaa2","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T21:11:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8964096-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dhxkp\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-dhxkp (namespace arobit) on cluster ci01-j8964096-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840280725065728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7114,"pr_state":"closed","pr_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","final_merged_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:19:26Z","started_at":"2026-09-23T19:19:26Z","completed_at":"2026-09-23T21:36:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:51:43.844260175Z","timing_metadata_checked_at":"2026-09-23T21:51:43.844260175Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:51:43.847559664Z","region_metadata_checked_at":"2026-09-23T21:51:43.847559664Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102835259199459328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"932144cafaab4d6ac2c6ccac83cf8e59c8e51edb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T18:59:29Z","started_at":"2026-09-23T18:59:29Z","completed_at":"2026-09-23T21:12:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:20:50.20722657Z","timing_metadata_checked_at":"2026-09-23T21:20:50.20722657Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:20:50.211184557Z","region_metadata_checked_at":"2026-09-23T21:20:50.211184557Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"open","pr_sha":"f0dfa64a174a3416406d82d8241fb7426d3e8153","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T18:35:44Z","started_at":"2026-09-23T18:35:44Z","completed_at":"2026-09-23T20:54:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.592813505Z","timing_metadata_checked_at":"2026-09-23T20:59:01.592813505Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.595513696Z","region_metadata_checked_at":"2026-09-23T20:59:01.595513696Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"59c64f763f8131224bd3746a78983b1dd5e6ac20fc51b52a32f73d52f809c9f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","failed_at":"alert","signature_id":"312cea2b73363d1b160f9cce1d956a42a926ffe6d5a41922d416b5287eec53f4","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T20:25:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4862464-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6s587\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-6s587 (namespace arobit) on cluster ci01-j4862464-svc has averaged 125.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"a8243d83bccfa56e550504ecf298a6d5ea6037918b8a194396659f6167827d27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","failed_at":"alert","signature_id":"766b1d46f5072f7c906c6126dd0836e5f198da05367c355a94ce2d777ad7547e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T19:49:29Z\n  Ended:       2026-09-23T20:23:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4862464-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4862464-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825931050586112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"open","pr_sha":"1ff09f1b1c644ee60c41a9b6ce0baafd2280068e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T18:22:25Z","started_at":"2026-09-23T18:22:25Z","completed_at":"2026-09-23T20:48:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.732982061Z","timing_metadata_checked_at":"2026-09-23T20:59:01.732982061Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.736108951Z","region_metadata_checked_at":"2026-09-23T20:59:01.736108951Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825299610701824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T18:19:54Z","started_at":"2026-09-23T18:19:54Z","completed_at":"2026-09-23T18:48:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:12.010947298Z","timing_metadata_checked_at":"2026-09-23T18:52:12.010947298Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T18:52:12.013930388Z","region_metadata_checked_at":"2026-09-23T19:54:40.427491975Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dc6040c2957dc298543424875eafcc0af051fdf89d7f2d91edd92e7e0c186e60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825299610701824","occurred_at":"2026-09-23T18:19:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"30be60dc7d1a742de6c8ee95366914897d94f6eb","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T18:09:20Z","started_at":"2026-09-23T18:09:20Z","completed_at":"2026-09-23T20:17:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:27:19.234682177Z","timing_metadata_checked_at":"2026-09-23T20:27:19.234682177Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:27:19.237546368Z","region_metadata_checked_at":"2026-09-23T20:27:19.237546368Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fc2abd0d1661a3c4ae4e0fd028ce0ce09f68f71c2ec1759f04869dc5bb5a2a90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","occurred_at":"2026-09-23T18:09:20Z","failed_at":"e2e","signature_id":"a7690e671288365fae42a54239a1fd3d4683bfffda4c651366b6b6d63b190ce6","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T17:54:24Z","started_at":"2026-09-23T17:54:24Z","completed_at":"2026-09-23T19:55:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:05:32.713251638Z","timing_metadata_checked_at":"2026-09-23T20:05:32.713251638Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:05:32.716568127Z","region_metadata_checked_at":"2026-09-23T20:05:32.716568127Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8cfbcb69b92bcf646b23ae81033bebb5fcd9f63391da41a8edf52e3429f3cab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","occurred_at":"2026-09-23T17:54:24Z","failed_at":"e2e","signature_id":"d95e88444841564bfebc7b4d86984dd9a38294a94ca9cf72a10f4adf98342abd","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006be420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n      \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006be420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n      \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:52:29Z","started_at":"2026-09-23T17:52:29Z","completed_at":"2026-09-23T20:10:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:16:28.618192962Z","timing_metadata_checked_at":"2026-09-23T20:16:28.618192962Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:16:28.622704146Z","region_metadata_checked_at":"2026-09-23T20:16:28.622704146Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a15f14399161f97da2ef8a309986d00cdc999dc3e5acb7da164323732285c68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","occurred_at":"2026-09-23T17:52:29Z","failed_at":"e2e","signature_id":"b269b2a196f9ad9cd1702b095e9f309573b62770b7cab43403d5312cc9615be1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:49:53Z","started_at":"2026-09-23T17:49:53Z","completed_at":"2026-09-23T18:43:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:12.188337813Z","timing_metadata_checked_at":"2026-09-23T18:52:12.188337813Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:12.190782405Z","region_metadata_checked_at":"2026-09-23T18:52:12.190782405Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"387b14b8dd4f9e51dd930a4fe612d59829a9353b0379e06c85422427cd165d8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","occurred_at":"2026-09-23T17:49:53Z","failed_at":"provision","signature_id":"aabbcc585420cff60362100bdaf09336ff0307d5c296c6855ec0f5fe21f850ce","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T18:34:52.490Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 values=\"map[cloudEnvironmentName:AzurePublicCloud cosmos:map[name:arohcpci01-rp-j6376960 url:https://arohcpci01-rp-j6376960.documents.azure.com:443/] image:map[digest:sha256:6bc5ed31e42afccc2abd6b21b817cc402be829c20f8cc00e71fde01c9be1523a registry:arohcpsvcdev.azurecr.io repository:ci-op-6hlkxni0/pipeline] registration:map[aksResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j6376960-mgmt-1 autoApprove:true hostedClustersManagedIdentitiesKeyVaultURL:https://ah-ci01-mi-j6376960-1.vault.azure.net/ hostedClustersSecretsKeyVaultManagedIdentityClientID:b2cc94cd-3e8b-42a9-bf2a-e025837a95cd hostedClustersSecretsKeyVaultURL:https://ah-ci01-cx-j6376960-1.vault.azure.net/ kubeApplierCosmosContainerName:Manifests-MC-1 maestroConsumerName:hcp-underlay-aro-hcp-dev-shard2-slot-01-mgmt-1 maestroGRPCTarget:maestro-grpc.maestro.svc.cluster.local:8090 maestroRESTAPIURL:http://maestro.maestro.svc.cluster.local:8000 provisionShardStatus:active publicDNSZoneResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960/providers/Microsoft.Network/dnsZones/aro-hcp-dev-shard2-slot-01.hcp.osadev.cloud stampIdentifier:1] serviceAccountName:fleet]\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:34:54.319Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 namespace=fleet\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 dryRun=false\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:35:05.401Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 numVersions=0\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"\\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38","failure_pattern":"failed to roll out Helm release: \"fleet-reg-ci01-j6376960-mgmt-1\" has no deployed releases","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T17:28:39Z","started_at":"2026-09-23T17:28:39Z","completed_at":"2026-09-23T19:43:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:54:40.593347954Z","timing_metadata_checked_at":"2026-09-23T19:54:40.593347954Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:54:40.597498241Z","region_metadata_checked_at":"2026-09-23T19:54:40.597498241Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5c82024eb87d47d5728ca00a2482dc960b51279e5d62991eafaaeb3b7bb0e73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","occurred_at":"2026-09-23T17:28:39Z","failed_at":"alert","signature_id":"c1db43e1f9ed556a1e717aa9ea521e513132026799cb40e29c7cc1976942dff0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T18:53:46Z\n  Ended:       2026-09-23T19:08:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j5312512-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j5312512-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7117,"pr_state":"closed","pr_sha":"ea43f7ba6df9f089661d7c30a490eb1e026104e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:26:00Z","started_at":"2026-09-23T17:26:00Z","completed_at":"2026-09-23T19:41:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:54:40.770229997Z","timing_metadata_checked_at":"2026-09-23T19:54:40.770229997Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:54:40.772861789Z","region_metadata_checked_at":"2026-09-23T19:54:40.772861789Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"27ff72046bb52fb7d0d7fbc701b6fa3eff5233a11846607241a6305c9bf68f8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"e2e","signature_id":"23493215ee1a820f0efaa9de36d3f924d4bf22faf75dcf25d674d9fd4bee714f","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}},{"row_id":"2f62dd92bed9e66582eab23ab554d6abf339c407e81aa301bfcb65cbe160a27a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"alert","signature_id":"cedc29dd595c13bb38b5f8171952695156bf1b2712e54cef49d283d640a3a08e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T19:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4456960-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ltf9x\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-ltf9x (namespace arobit) on cluster ci01-j4456960-svc has averaged 125% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"717bd2a6551a73e19b4c02e2e56f8d8bc799bdd4e4439f1455907bab661aabd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"alert","signature_id":"56b2ff341978727c646e219e23fba2dfc1ef2e5ccc4828702e0dece8d2810ba6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T18:28:51Z\n  Ended:       2026-09-23T19:02:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4456960-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4456960-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"8402bcb01e7cb18edcf2352d8e3571f186edf06483a251f462bf537f1c95351d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"e2e","signature_id":"48a8490fb0e062a5d2a53db5bf3f5a92fd4adef796973aafbfa795c3fe0dbe08","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a21860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n      \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a21860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n      \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:43:09Z","started_at":"2026-09-23T16:43:09Z","completed_at":"2026-09-23T18:51:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:03:01.998237305Z","timing_metadata_checked_at":"2026-09-23T19:03:01.998237305Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:03:02.005257782Z","region_metadata_checked_at":"2026-09-23T19:03:02.005257782Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c9d8a130d91ea03de00bcfd1d700efb07ab41edf36fe56ee29600f68dab88c04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","occurred_at":"2026-09-23T16:43:09Z","failed_at":"alert","signature_id":"f1b24a151cda52cd0d54ea6a4677f8fb34ef83c5bdadd527eb334c9d72e78bde","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T18:30:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9325440-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nc4vq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-nc4vq (namespace arobit) on cluster ci01-j9325440-svc has averaged 124% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7114,"pr_state":"closed","pr_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","final_merged_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:40:45Z","started_at":"2026-09-23T16:40:45Z","completed_at":"2026-09-23T18:41:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.701287242Z","timing_metadata_checked_at":"2026-09-23T18:52:13.701287242Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.704425432Z","region_metadata_checked_at":"2026-09-23T18:52:13.704425432Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"297538984fb50a279dfb92919b5102e540682b0dd82696d8c2cfa2bf65f43fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","occurred_at":"2026-09-23T16:40:45Z","failed_at":"e2e","signature_id":"8a478233ef9f4819c24a5dc34485444f654a99c79dc829ccb980b0bc2625e398","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b97da0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8g5rnldzw776\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"name\": \"607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:42:05.950445215Z\",\n      \"endTime\": \"2026-09-23T18:01:06.854588111Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b97da0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8g5rnldzw776\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"name\": \"607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:42:05.950445215Z\",\n      \"endTime\": \"2026-09-23T18:01:06.854588111Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:39:42Z","started_at":"2026-09-23T16:39:42Z","completed_at":"2026-09-23T18:45:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.557195214Z","timing_metadata_checked_at":"2026-09-23T18:52:13.557195214Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.559930305Z","region_metadata_checked_at":"2026-09-23T18:52:13.559930305Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f0d674dbebb4c47bc310791ffaaa65927e17896ff0783e6c30b1c633998e8f0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","occurred_at":"2026-09-23T16:39:42Z","failed_at":"alert","signature_id":"aca35d9e741dfeba834ff114bffafa2a455703b5a32f415b37bdcbd261eb6a67","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T17:51:27Z\n  Ended:       2026-09-23T18:00:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1331712-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1331712-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102799578414714880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"466e2e9244431d18141c0e26224633aa9f8122d0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T16:37:42Z","started_at":"2026-09-23T16:37:42Z","completed_at":"2026-09-23T18:40:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.42681784Z","timing_metadata_checked_at":"2026-09-23T18:52:13.42681784Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.429739731Z","region_metadata_checked_at":"2026-09-23T18:52:13.429739731Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795320147578880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T16:20:46Z","started_at":"2026-09-23T16:20:46Z","completed_at":"2026-09-23T18:22:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.349049248Z","timing_metadata_checked_at":"2026-09-23T18:31:59.349049248Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.356817724Z","region_metadata_checked_at":"2026-09-23T18:31:59.356817724Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:20:35Z","started_at":"2026-09-23T16:20:35Z","completed_at":"2026-09-23T18:27:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.528363004Z","timing_metadata_checked_at":"2026-09-23T18:31:59.528363004Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.530941296Z","region_metadata_checked_at":"2026-09-23T18:31:59.530941296Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b6260ae90c2cc05786ae25a4e151ab0ca6388740039250e349fe5f4152a35151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","occurred_at":"2026-09-23T16:20:35Z","failed_at":"e2e","signature_id":"2562b6a0aebd3dafa3041bae2bbef4d61a7d09096d6b568a3e7b98b9b1ca7da6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001018240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n      \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001018240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n      \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:20:32Z","started_at":"2026-09-23T16:20:32Z","completed_at":"2026-09-23T18:28:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:42:06.771021902Z","timing_metadata_checked_at":"2026-09-23T18:42:06.771021902Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:42:06.773865093Z","region_metadata_checked_at":"2026-09-23T18:42:06.773865093Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"54138909a11d3ecdba2bce14c613918e2342ca0a5592cb34205416ab457a16e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","occurred_at":"2026-09-23T16:20:32Z","failed_at":"alert","signature_id":"c0de1a0ccf1f7966d446e232f88e4d90f26f4017b261cdeb15d15f7eb77aa118","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T17:46:48Z\n  Ended:       2026-09-23T17:57:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9858944-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-6f4c9cc744-8nwkb\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-6f4c9cc744-8nwkb (namespace kube-applier) on cluster ci01-j9858944-mgmt-2 has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102794564984115200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7077,"pr_state":"closed","pr_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","final_merged_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T16:17:46Z","started_at":"2026-09-23T16:17:46Z","completed_at":"2026-09-23T18:21:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.625986608Z","timing_metadata_checked_at":"2026-09-23T18:31:59.625986608Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.629440298Z","region_metadata_checked_at":"2026-09-23T18:31:59.629440298Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102792596072960000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:09:57Z","started_at":"2026-09-23T16:09:57Z","completed_at":"2026-09-23T16:23:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.825071746Z","timing_metadata_checked_at":"2026-09-23T16:36:58.825071746Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.827780238Z","region_metadata_checked_at":"2026-09-23T16:36:58.827780238Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c8ce51fae56024717e83ccbad328bfaa255c5b2676f6e1f3eb8d15bf1e24b047","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102792596072960000","occurred_at":"2026-09-23T16:09:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:00:38Z","started_at":"2026-09-23T16:00:38Z","completed_at":"2026-09-23T17:54:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:00:55.400632312Z","timing_metadata_checked_at":"2026-09-23T18:00:55.400632312Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:00:55.4041462Z","region_metadata_checked_at":"2026-09-23T18:00:55.4041462Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"772a1aecdfe2ebf92e56874be2be523e71a6adc38da8e579c0a229365db4f93e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","occurred_at":"2026-09-23T16:00:38Z","failed_at":"e2e","signature_id":"2074bff381a20809ab3d0b56df62ec1c9c4cecad07bc3b02a8f9d837a90a1b6a","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506","failure_pattern":"ERROR CODE: InternalServerError; context failed to retrieve serial console logs for VM \"\u003cvm\u003e\"","search_query":"\"message\": \"Internal server error.\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"b542c3a7da99e0f09c6b2e4cfc2b3cb9c2bc7980","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T15:31:45Z","started_at":"2026-09-23T15:31:45Z","completed_at":"2026-09-23T17:30:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:40:02.323890462Z","timing_metadata_checked_at":"2026-09-23T17:40:02.323890462Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:40:02.326927052Z","region_metadata_checked_at":"2026-09-23T17:40:02.326927052Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0a3c0d9132c6ae9b63e97c3f8e749422c5e0106c72c7efe095e188b7763ffa32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","failed_at":"e2e","signature_id":"0c29798c8c1659484795e61241cd7388e309cd731fe979947b39b617cfeb82bb","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"bfc85548cb54340c8476dd84cd1decc1b60e8bc751272b29c6f37a41f91f38cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:33:04Z\n  Ended:       2026-09-23T16:39:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4918656-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"efbaa775509498dd316f673975db4418a30ad62f","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:56:11Z","started_at":"2026-09-23T14:56:11Z","completed_at":"2026-09-23T16:56:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:07:59.287530719Z","timing_metadata_checked_at":"2026-09-23T17:07:59.287530719Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:07:59.293631401Z","region_metadata_checked_at":"2026-09-23T17:07:59.293631401Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"da7a8a0fd146ac84a973b80124dc195a9733bcc594167fcb4c95302d05b117d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","occurred_at":"2026-09-23T14:56:11Z","failed_at":"e2e","signature_id":"3812816f7f5de01152a0f4bc5ce516ed73308e2f6080e009a2d35e56b303c38c","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:54:53Z","started_at":"2026-09-23T14:54:53Z","completed_at":"2026-09-23T17:02:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:07:59.157356622Z","timing_metadata_checked_at":"2026-09-23T17:07:59.157356622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:07:59.160642612Z","region_metadata_checked_at":"2026-09-23T17:07:59.160642612Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"a0f821b907c42632e10ca7689b83f85ea79052298bc9b9a9f777da7ecfb844fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"e2e","signature_id":"e0cff06d2d3890e9b05a9d35c28b2aff9003a5f93ca966b808d2a0e175a745b6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad4c3242785602103ddf9d0aa115d02dae9c08f9551055e43c550435cf51369b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"alert","signature_id":"d57782e3eb6f6cd2059f92437161a764f82ac19056eed7d813da72181ad2f92d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:15:00Z\n  Ended:       2026-09-23T16:46:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T16:15:00Z\n  Ended:       2026-09-23T16:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"c5c269a24369c046230bfcdb2787ff2e337a13aeb3a8ba23daf31c87ba78b9a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"alert","signature_id":"d679a18a9e7a13bb82f0cbe4fdbabac5624367c05fd9b68e2584227db8962e2f","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:17:56Z\n  Ended:       2026-09-23T16:44:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n  Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T16:17:56Z\n  Ended:       2026-09-23T16:45:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n  Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"d8e65efa8b4362aa79603b17c2892fea71ff8cdff9983290761af55222c5e2f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"e2e","signature_id":"19b631f7d5f2a850bd1bdac6e41017fbcf045a75d8474e6adf13f1324cd43c53","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7087/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773191775621120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7087,"pr_state":"closed","pr_sha":"efdd1a00e3bd03f1bb647856387c92cf040a29fe","final_merged_sha":"6ef108e37293c8a1a05b1ee1679fda670a90bb09","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T14:52:51Z","started_at":"2026-09-23T14:52:51Z","completed_at":"2026-09-23T17:05:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:18:34.497010397Z","timing_metadata_checked_at":"2026-09-23T17:18:34.497010397Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:18:34.500430386Z","region_metadata_checked_at":"2026-09-23T17:18:34.500430386Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T14:50:10Z","started_at":"2026-09-23T14:50:10Z","completed_at":"2026-09-23T17:09:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:18:34.658262464Z","timing_metadata_checked_at":"2026-09-23T17:18:34.658262464Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:18:34.660786255Z","region_metadata_checked_at":"2026-09-23T17:18:34.660786255Z"},"failed_at":["alert","e2e"],"failed_test_count":26,"occurrences":[{"row_id":"03aab9bc365cc534c7752ced0f5debd4812f44e341f4220f097043b0cc3c9e83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"fb3c31a801804fd94e8c51eb7bcff2f2193b26a3f6b7ab9373ac03f4a1eac932","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--q72fthqnzbpj/cluster-hs-pre-hbzxm7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b10260\u003e: \n    failed to create HCP cluster cluster-hs-pre-hbzxm7: failed waiting for cluster=\"cluster-hs-pre-hbzxm7\" in resourcegroup=\"rg-hs-pre--q72fthqnzbpj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"name\": \"e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.68921584Z\",\n      \"endTime\": \"2026-09-23T16:22:46.778697738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--q72fthqnzbpj/cluster-hs-pre-hbzxm7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b10260\u003e: \n    failed to create HCP cluster cluster-hs-pre-hbzxm7: failed waiting for cluster=\"cluster-hs-pre-hbzxm7\" in resourcegroup=\"rg-hs-pre--q72fthqnzbpj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"name\": \"e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.68921584Z\",\n      \"endTime\": \"2026-09-23T16:22:46.778697738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"04a2bba35bdf5f31e066c03c20697bc9ec614b3ee4092efd3acce7ace7b16334","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"2df52c1cafa5532c454b36cb7b084b68593cebfadd2864d45dd1a118200b8d52","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jmdbzj\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0980\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jmdbzj: failed waiting for cluster=\"cluster-zstream-4-21-jmdbzj\" in resourcegroup=\"rg-zstream-upgrade-4-21-4gvzx8nqmkwd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"name\": \"6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.347085548Z\",\n      \"endTime\": \"2026-09-23T16:22:56.388404215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jmdbzj\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0980\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jmdbzj: failed waiting for cluster=\"cluster-zstream-4-21-jmdbzj\" in resourcegroup=\"rg-zstream-upgrade-4-21-4gvzx8nqmkwd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"name\": \"6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.347085548Z\",\n      \"endTime\": \"2026-09-23T16:22:56.388404215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"08ac124f535a759e298dd9594a0b329f0c12e627046279e746f2aa09b78480ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"449030cbb33da5d4c8dac990dceda44dd62ccc5ca1ffc1ccd12391d7b55e0681","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00055a3c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z85zj8vh8z7t\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"name\": \"fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:38.637505727Z\",\n      \"endTime\": \"2026-09-23T16:23:46.403891132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00055a3c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z85zj8vh8z7t\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"name\": \"fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:38.637505727Z\",\n      \"endTime\": \"2026-09-23T16:23:46.403891132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0dfb0368223a888f99738c68a538ce5d96639f292fab5e7abb9471b3f23bb771","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"b4f75fac021e402acc2db264f349c96a35364d535bee0b2e0065ea599920a92f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-t2gnbd\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe480\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-t2gnbd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-t2gnbd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-cbnzwcvqnmsl\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"name\": \"5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.790630075Z\",\n      \"endTime\": \"2026-09-23T16:22:56.599612998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-t2gnbd\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe480\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-t2gnbd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-t2gnbd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-cbnzwcvqnmsl\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"name\": \"5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.790630075Z\",\n      \"endTime\": \"2026-09-23T16:22:56.599612998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"251be9ebc0d377de84fb5bd489fb8ae8845df64822d2479dc40a0ae5d2b8c162","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"0adb4bb4ae9202e459b2315ccfce11f495585f3d7bac3fc7a2487d9796add3c7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005383e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lmfj4bczvz8s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"name\": \"13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.078676518Z\",\n      \"endTime\": \"2026-09-23T16:22:46.814369908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005383e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lmfj4bczvz8s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"name\": \"13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.078676518Z\",\n      \"endTime\": \"2026-09-23T16:22:46.814369908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"294bc2922170c7a1b61a069bae0a2886966661cc46b8b6ffbbb25543ef46b307","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"9e823b348b35d763f4f7278b55c97380508738106688d0cd3c61c152c24157ec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-hj788x\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f801c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-hj788x: failed waiting for cluster=\"cluster-zstream-4-22-hj788x\" in resourcegroup=\"rg-zstream-upgrade-4-22-jdz9qkrk7g87\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"name\": \"cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.267758855Z\",\n      \"endTime\": \"2026-09-23T16:22:46.837301028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-hj788x\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f801c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-hj788x: failed waiting for cluster=\"cluster-zstream-4-22-hj788x\" in resourcegroup=\"rg-zstream-upgrade-4-22-jdz9qkrk7g87\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"name\": \"cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.267758855Z\",\n      \"endTime\": \"2026-09-23T16:22:46.837301028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d074f1de4e240b2a88bfafd50367f03c7eafd6672c012afe30affba9653d556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"e1d6b4b1a86d1907639ae72cc2ccc8137426d24a60ad5b4f6f3e7247ee41a9cc","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbc500\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb9nfmvpzmj4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\",\n      \"name\": \"0f138583-3da6-4491-9246-0255d290efc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.301957583Z\",\n      \"endTime\": \"2026-09-23T16:22:56.376646542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbc500\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb9nfmvpzmj4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\",\n      \"name\": \"0f138583-3da6-4491-9246-0255d290efc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.301957583Z\",\n      \"endTime\": \"2026-09-23T16:22:56.376646542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"342d5e0b4772c14f0ce24853aa1b44bc78d4a1dff23fe5440773775a6c0fbfad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"36a05e474622bc3fcc801d228958fffc4a6d075602e55266d09e5b2e91c19368","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-8w2jrzrjz9pw/cluster-candidate-4-20-wj5jwq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001648460\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wj5jwq: failed waiting for cluster=\"cluster-candidate-4-20-wj5jwq\" in resourcegroup=\"rg-candidate-4-20-8w2jrzrjz9pw\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"name\": \"52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:34.986564037Z\",\n      \"endTime\": \"2026-09-23T16:22:36.376773532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-8w2jrzrjz9pw/cluster-candidate-4-20-wj5jwq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001648460\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wj5jwq: failed waiting for cluster=\"cluster-candidate-4-20-wj5jwq\" in resourcegroup=\"rg-candidate-4-20-8w2jrzrjz9pw\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"name\": \"52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:34.986564037Z\",\n      \"endTime\": \"2026-09-23T16:22:36.376773532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3dd9bd246d279d2117f51e531b7a9812653113a1b3036d50424ecf374be7558c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"97bb982d1e6ea1aa19045a58c60ec413fe3ba3d8834e3141a02e5782483f6785","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f646a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cpzrksbzwbnx\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"name\": \"3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:20.371775923Z\",\n      \"endTime\": \"2026-09-23T16:24:26.21161142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f646a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cpzrksbzwbnx\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"name\": \"3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:20.371775923Z\",\n      \"endTime\": \"2026-09-23T16:24:26.21161142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f135409be34decbf7cf1ea979cd95f609cc68095d2a1f1685eab4b366ef8b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"2f81637f8265c258ea75aa24f9fd9432ef36114b9ebee46504be7d5770c435b7","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012448d0\u003e: \n    GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"name\": \"c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:30.756948595Z\",\n      \"endTime\": \"2026-09-23T16:23:36.211198776Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012448d0\u003e: \n    GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"name\": \"c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:30.756948595Z\",\n      \"endTime\": \"2026-09-23T16:23:36.211198776Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f79d82e16020a0c88a3e57a3fd3aec0b4911c2cbe6d6e9854a2e2199d232b31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"5f3ee90b05be0d7f1487912ea92f8d3f0441c3abfa713a796fa8f25021c270ac","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24340\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-pp5dtlw88rfs\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"name\": \"bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T16:17:57.447240329Z\",\n      \"endTime\": \"2026-09-23T16:47:00.702969424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24340\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-pp5dtlw88rfs\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"name\": \"bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T16:17:57.447240329Z\",\n      \"endTime\": \"2026-09-23T16:47:00.702969424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c0071eb8b13c2a5162e89bb1eb353a066789900906c50095ffabf420f916329","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"a465030c91e7749c08c2c7314b00f14fac042b380238eb18631f6fc0cc7ad978","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107a1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-z5kd6wr2c86f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"name\": \"5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.576887124Z\",\n      \"endTime\": \"2026-09-23T16:22:56.469457208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107a1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-z5kd6wr2c86f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"name\": \"5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.576887124Z\",\n      \"endTime\": \"2026-09-23T16:22:56.469457208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b51e268b6b1f576bbce735ca73aa0d9c4bceb44f5f967934b727dd9a08a783f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"dd70726fbf025a2e22838d8e9a9bcfaa30032d7ec07810dbb606326dc89fe874","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5cc20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8j8ndzvzp2dj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\",\n      \"name\": \"a2be5479-1530-4944-82f9-a854036654b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.658130554Z\",\n      \"endTime\": \"2026-09-23T16:22:56.230648382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5cc20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8j8ndzvzp2dj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\",\n      \"name\": \"a2be5479-1530-4944-82f9-a854036654b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.658130554Z\",\n      \"endTime\": \"2026-09-23T16:22:56.230648382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"67c53202595f7fc78fd7ba40f496859d9ba76c974210d862db260db22fa46ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"51f8c6265949434e6442ba80f7ade4730dcad12175f31515ce73c25444d377e5","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skmlmr with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00166a640\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skmlmr: failed waiting for cluster=\"np-version-upgrade-cluster-skmlmr\" in resourcegroup=\"rg-np-version-upgrade-skmlmr-gth9wdbm2lgd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"name\": \"edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.126846914Z\",\n      \"endTime\": \"2026-09-23T16:22:56.455577909Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skmlmr with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00166a640\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skmlmr: failed waiting for cluster=\"np-version-upgrade-cluster-skmlmr\" in resourcegroup=\"rg-np-version-upgrade-skmlmr-gth9wdbm2lgd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"name\": \"edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.126846914Z\",\n      \"endTime\": \"2026-09-23T16:22:56.455577909Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7185d8e7a8c0d094cc8529d8f0110487422e4c897c82ce5d3536e03cb8a3137f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"d6cea3ffddfab34ef89656bb4911dcfd537e070bc053f39a0b804e5db90f21f8","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012263c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-9v628njd5r4h\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"name\": \"eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.077341251Z\",\n      \"endTime\": \"2026-09-23T16:22:46.744916423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012263c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-9v628njd5r4h\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"name\": \"eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.077341251Z\",\n      \"endTime\": \"2026-09-23T16:22:46.744916423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bdcaacc0c17e5f2a6281ddd22c53324c5309cafe2b3c4a4995b9af4948a9b54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"89edcec5785330a2ee63b672917ff746b2f687ab7ded705d0cc185c95d8bec14","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j694t442mj2r\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"name\": \"f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.255245987Z\",\n      \"endTime\": \"2026-09-23T16:22:56.363291103Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j694t442mj2r\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"name\": \"f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.255245987Z\",\n      \"endTime\": \"2026-09-23T16:22:56.363291103Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82732f43ba874be54d8912d3278daebd2f13398829cde0bd596f126c4b26fa3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"9731f91656941bbb9380fa37cca11d45aa6ad2c56b1236e28cb4401092e4c0e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gt4scq at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018268c0\u003e: \n    failed to create HCP cluster np-skip-minor-gt4scq: failed waiting for cluster=\"np-skip-minor-gt4scq\" in resourcegroup=\"rg-np-skip-minor-gt4scq-f9nsllw59lzt\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\",\n      \"name\": \"949773e8-36e8-45db-a78e-a47066415d62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.566834881Z\",\n      \"endTime\": \"2026-09-23T16:22:56.625731333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gt4scq at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018268c0\u003e: \n    failed to create HCP cluster np-skip-minor-gt4scq: failed waiting for cluster=\"np-skip-minor-gt4scq\" in resourcegroup=\"rg-np-skip-minor-gt4scq-f9nsllw59lzt\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\",\n      \"name\": \"949773e8-36e8-45db-a78e-a47066415d62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.566834881Z\",\n      \"endTime\": \"2026-09-23T16:22:56.625731333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f4a3aa0be444344d70bf52d8eb245aae2b3a86dfd63d4d9d249afe7b78a6112","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"alert","signature_id":"e210a4373a8a1dff8335c12bcbfb22a4bca9ffd1f73200c066c63547ac34373a","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:16:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:16:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"922d61aa5578a722be8242cb496c0e263e1b77a2933ad39ee3f4592103e77f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"alert","signature_id":"0b112a92c68becda58d472a3dbc17b517a7589cf579446aec092f675af664746","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:26:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:27:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"a7ba0c727b565e597e877e0d9934aa33a08b0d4c82df6eb91de1fa3414c8aab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"cfc810c40e60a646985d0ea21bde9b0fc719a1179e342cbce3fe6eb38a62ed69","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7gd6nm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f4660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7gd6nm: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7gd6nm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rrnk84sgmlwn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\",\n      \"name\": \"77c395bf-c417-4455-9807-799065c6b12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.926344123Z\",\n      \"endTime\": \"2026-09-23T16:22:56.209934575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7gd6nm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f4660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7gd6nm: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7gd6nm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rrnk84sgmlwn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\",\n      \"name\": \"77c395bf-c417-4455-9807-799065c6b12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.926344123Z\",\n      \"endTime\": \"2026-09-23T16:22:56.209934575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7f5b5cd255469615d7fdc4c94cd0e36d30cd6fd18157b18d7cc43e208f068b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"6fc513af79b2809dcb5d5df1bdafeac7cf5d4c3a1862412ed3eba47ccae158c5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001164480\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-sg68hlrsd9c4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"name\": \"c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.818708285Z\",\n      \"endTime\": \"2026-09-23T16:22:46.588447682Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001164480\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-sg68hlrsd9c4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"name\": \"c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.818708285Z\",\n      \"endTime\": \"2026-09-23T16:22:46.588447682Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c2d97b03a7e81dee96c9c83141626dd2ccb4c44948fc9e7f0ed27e8fe0a7f004","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"1b19c6dab2da849dbdc19f25d8a7c8a64dadf6697567bfe5d10096294dd0a6e4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4240\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tj58cr2mq6ls\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"name\": \"eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.408421024Z\",\n      \"endTime\": \"2026-09-23T16:22:46.864374297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4240\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tj58cr2mq6ls\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"name\": \"eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.408421024Z\",\n      \"endTime\": \"2026-09-23T16:22:46.864374297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"da7bbf8d0c6b951ab9d14abb2bd01d2eef11fa33ff3532e0fa596c88d9710e76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"381000ecc9d6086c37b0219505b21d93a9c939a2ccf23f03576b38ab84b5a55e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d141e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9lh7k5m447kj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"name\": \"02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.505483407Z\",\n      \"endTime\": \"2026-09-23T16:22:56.544110064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d141e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9lh7k5m447kj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"name\": \"02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.505483407Z\",\n      \"endTime\": \"2026-09-23T16:22:56.544110064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"db593533e7296bc60d8b9c297d7d75897dcdaaade801ff1656e4d37de1021d90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"cb7663c2933e4c98c3163332183af2e5279eff6073b28c84e67ab60d8693fc1a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-5vk9ph\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017a2760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-5vk9ph: failed waiting for cluster=\"cp-ystream-upgrade-4-22-5vk9ph\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-dw4krrwmbz6s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"name\": \"c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.801712787Z\",\n      \"endTime\": \"2026-09-23T16:22:56.470110002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-5vk9ph\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017a2760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-5vk9ph: failed waiting for cluster=\"cp-ystream-upgrade-4-22-5vk9ph\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-dw4krrwmbz6s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"name\": \"c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.801712787Z\",\n      \"endTime\": \"2026-09-23T16:22:56.470110002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd2a5fdb09400a11ae9c34643ca8e1c913342005c5c0611d1f1f154e5ce1a897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"e94f7132ddfa2da930718c7e555576608ef511972be7b068ecd1cbc93745767d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"47903548f1f143d7ab9998fc3c8c1fbc469323e86e2d9a0d0fddfcc0a731af5d","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f98140\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l95n8mx296wc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"name\": \"7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.243780388Z\",\n      \"endTime\": \"2026-09-23T16:22:56.457631227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f98140\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l95n8mx296wc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"name\": \"7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.243780388Z\",\n      \"endTime\": \"2026-09-23T16:22:56.457631227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":26,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":26,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:36:53Z","started_at":"2026-09-23T14:36:53Z","completed_at":"2026-09-23T16:33:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:47:21.245116319Z","timing_metadata_checked_at":"2026-09-23T16:47:21.245116319Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:47:21.249519704Z","region_metadata_checked_at":"2026-09-23T16:47:21.249519704Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6a99f29f78230bf3d45bba48fa2f5456a75e1fe745b14b6136b119cfa5954394","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","occurred_at":"2026-09-23T14:36:53Z","failed_at":"alert","signature_id":"3135ce76bbdd1ef05af982db7996ed6329db94eadabfb94067b89a304dc4d2cb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T15:53:20Z\n  Ended:       2026-09-23T15:59:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0727168-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0727168-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102765375618813952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T14:21:41Z","started_at":"2026-09-23T14:21:41Z","completed_at":"2026-09-23T16:23:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.688836874Z","timing_metadata_checked_at":"2026-09-23T16:36:58.688836874Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.693511359Z","region_metadata_checked_at":"2026-09-23T16:36:58.693511359Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"1769127263098290ba69f6f4e9bc0696f1c66e33","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:18:47Z","started_at":"2026-09-23T14:18:47Z","completed_at":"2026-09-23T16:25:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.525309487Z","timing_metadata_checked_at":"2026-09-23T16:36:58.525309487Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.529756973Z","region_metadata_checked_at":"2026-09-23T16:36:58.529756973Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"20161848cb282b3d3cba8f3cb394772ad2a3d1f7b2ce18d2cb0e5a31e2806816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","occurred_at":"2026-09-23T14:18:47Z","failed_at":"alert","signature_id":"df2c8e2eea124fae55058fbdd964feff88c2979d18d038ec6b6702d5cd247db8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:11:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5116032-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cmbd9\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-cmbd9 (namespace arobit) on cluster ci01-j5116032-svc has averaged 125.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:11:30Z","started_at":"2026-09-23T14:11:30Z","completed_at":"2026-09-23T16:08:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:16:10.214826775Z","timing_metadata_checked_at":"2026-09-23T16:16:10.214826775Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:16:10.222905748Z","region_metadata_checked_at":"2026-09-23T16:16:10.222905748Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"46528e9037349e5cf8dc5a037258af3070882d93ced3d15bfac53af4ca0b7bc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","occurred_at":"2026-09-23T14:11:30Z","failed_at":"e2e","signature_id":"8c22befe458ceb42a7ccbeae384be919b61714799a47b6e0ea785ee3e8137ae4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118c460\u003e: \n    DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118c460\u003e: \n    DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:56:43Z","started_at":"2026-09-23T13:56:43Z","completed_at":"2026-09-23T16:04:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:16:10.515642378Z","timing_metadata_checked_at":"2026-09-23T16:16:10.515642378Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:16:10.521513258Z","region_metadata_checked_at":"2026-09-23T16:16:10.521513258Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7f32a2d651aa9c59386e454937a009e1b3b1c29a5395e0236fb4054d5b49e0f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","occurred_at":"2026-09-23T13:56:43Z","failed_at":"alert","signature_id":"b522a7cea6c2812e29c1ea3b0132f66a43bf87236c4f0123b2bd527a73ab14bb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T14:55:12Z\n  Ended:       2026-09-23T15:28:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3326976-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3326976-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102754197291143168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"4ac6a43f989e509deee6e8730933499503439b9a","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:37:22Z","started_at":"2026-09-23T13:37:22Z","completed_at":"2026-09-23T14:28:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:13:19.788575948Z","timing_metadata_checked_at":"2026-09-23T15:13:19.788575948Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:13:19.792359636Z","region_metadata_checked_at":"2026-09-23T15:13:19.792359636Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9c06f43823dd9c82f07acdd590da87ade7f804136d892954c0b278422e2bb16e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102754197291143168","occurred_at":"2026-09-23T13:37:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"11fda0e55e54db52516402aac997f9791cea0a7e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:34:43Z","started_at":"2026-09-23T13:34:43Z","completed_at":"2026-09-23T14:23:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:31:08.904547879Z","timing_metadata_checked_at":"2026-09-23T14:31:08.904547879Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:31:08.908040767Z","region_metadata_checked_at":"2026-09-23T14:31:08.908040767Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b581ef52a3a90c861844caae9cf6ca2f3bef9efb41e403e75021ea63a666da3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","occurred_at":"2026-09-23T13:34:43Z","failed_at":"provision","signature_id":"15f55de3f2ea759d42d39cc08a545eae0c5c976acedd44ef9735b3b78e99695d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T14:16:23.495Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-23T14:17:09.991Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb","failure_pattern":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102752015808466944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"a9f744ec99dd74bed140edab47cb30ee67d3912e","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T13:28:42Z","started_at":"2026-09-23T13:28:42Z","completed_at":"2026-09-23T15:29:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:44:33.291890491Z","timing_metadata_checked_at":"2026-09-23T15:44:33.291890491Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:44:33.29515818Z","region_metadata_checked_at":"2026-09-23T15:44:33.29515818Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:26:28Z","started_at":"2026-09-23T13:26:28Z","completed_at":"2026-09-23T15:50:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:05:43.007493418Z","timing_metadata_checked_at":"2026-09-23T16:05:43.007493418Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:05:43.011317406Z","region_metadata_checked_at":"2026-09-23T16:05:43.011317406Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"45b7e31d2e54d9afb213bf465c0e350a2adc2dde0b5c558156e356b62eada9d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","occurred_at":"2026-09-23T13:26:28Z","failed_at":"alert","signature_id":"27fba5395347858a8e8e715d32f7f0fe022ff7065fe1ea71ae9a42e0ed9a0a0d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T15:27:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1372160-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kr2cs\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-kr2cs (namespace arobit) on cluster ci01-j1372160-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7106,"pr_state":"closed","pr_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","final_merged_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:26:24Z","started_at":"2026-09-23T13:26:24Z","completed_at":"2026-09-23T15:32:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:44:33.450260276Z","timing_metadata_checked_at":"2026-09-23T15:44:33.450260276Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:44:33.453626065Z","region_metadata_checked_at":"2026-09-23T15:44:33.453626065Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d2d373686094856f7bdfad8d8d08fdef61c112b681b3c44a34d791d8d9e069b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","occurred_at":"2026-09-23T13:26:24Z","failed_at":"e2e","signature_id":"e4012e513d3cafa3f6c8980450c912f0d288e7cf2e0c893549e7a66042041837","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2ede0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2jl588bhfvmd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"name\": \"8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:47:29.897134227Z\",\n      \"endTime\": \"2026-09-23T15:06:35.791795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2ede0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2jl588bhfvmd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"name\": \"8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:47:29.897134227Z\",\n      \"endTime\": \"2026-09-23T15:06:35.791795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751270510006272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5493,"pr_state":"open","pr_sha":"10f8327910e5d5309d012a4c304ebcb9f2b4aa74","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T13:25:44Z","started_at":"2026-09-23T13:25:44Z","completed_at":"2026-09-23T15:26:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:33:49.759341373Z","timing_metadata_checked_at":"2026-09-23T15:33:49.759341373Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:33:49.762959061Z","region_metadata_checked_at":"2026-09-23T15:33:49.762959061Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:22:24Z","started_at":"2026-09-23T13:22:24Z","completed_at":"2026-09-23T15:23:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:33:49.938102164Z","timing_metadata_checked_at":"2026-09-23T15:33:49.938102164Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:33:49.941253354Z","region_metadata_checked_at":"2026-09-23T15:33:49.941253354Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0a3f8e613d4d3ef59dc621da97f419de2b26a9953865953aded0a19e9cf54e06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","failed_at":"e2e","signature_id":"8eb0ca7714bb43bcafee5fa2084b2284a94351ac2160b6d55e893d478c2e48d8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094e860\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-nkx9tc\" in resourcegroup=\"rg-np-skip-minor-nkx9tc-svtsnwjlpglh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"name\": \"2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:30:28.639210211Z\",\n      \"endTime\": \"2026-09-23T14:49:28.972470638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094e860\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-nkx9tc\" in resourcegroup=\"rg-np-skip-minor-nkx9tc-svtsnwjlpglh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"name\": \"2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:30:28.639210211Z\",\n      \"endTime\": \"2026-09-23T14:49:28.972470638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7378e7edfe7142a029cf5bd8d457c6e7e240863e3b7839d64527b18ccd60c2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","failed_at":"e2e","signature_id":"bccb01e525eebedfd6d8ea72bf2f185117e64e499bd98913c911974096d95997","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n      \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n      \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"4550f6295f01e13c8a713541787499b19d694a06","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T12:59:37Z","started_at":"2026-09-23T12:59:37Z","completed_at":"2026-09-23T14:49:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:02:24.20785241Z","timing_metadata_checked_at":"2026-09-23T15:02:24.20785241Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:02:24.2110087Z","region_metadata_checked_at":"2026-09-23T15:02:24.2110087Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ffe3d7b13f537077ac921f55036dfa0b03ee476b99c25331413cfdb82548e3b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","occurred_at":"2026-09-23T12:59:37Z","failed_at":"alert","signature_id":"f10cdf2980f54cb7434c563cea5c00835d86c81de376364c86add2439c33d20f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T13:47:46Z\n  Ended:       2026-09-23T14:19:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7183104-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7183104-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"open","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T12:35:31Z","started_at":"2026-09-23T12:35:31Z","completed_at":"2026-09-23T14:43:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:52:08.968389428Z","timing_metadata_checked_at":"2026-09-23T14:52:08.968389428Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:52:08.971641217Z","region_metadata_checked_at":"2026-09-23T14:52:08.971641217Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2ef9840825001e2a8318c3fc1eec97ce75bc6c2b6d3dcbb27256cb3c7d371262","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"alert","signature_id":"e8f9ea179e852213d3a81d66ef232d1e14041b2de7883a9afe6b09a4073f601b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T14:09:57Z\n  Ended:       2026-09-23T14:15:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6650496-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6650496-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9752d5ac5dd7ded0de4eef211c7c8c920a8115c6c7a8088e373f4aa67e655663","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"e2e","signature_id":"0f9515a2db56a29e5a383d5fa728e947a2b0cee316b599fe636e25ce8c008556","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n      \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n      \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp"}},{"row_id":"b5e5c1987a3648f8cc4f9fd2d02338b5cd1515495ded580221f4180c7dcafed1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"alert","signature_id":"c5b74c40552ae1b36fab99026dc41a2a915f1d1a792b500821eca9610ee3d31e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T14:29:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6650496-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wqp86\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-wqp86 (namespace arobit) on cluster ci01-j6650496-svc has averaged 125% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"ed52a45a00107d337dccfc29cf9418d334900ca6659eeabc51ebc9dfcbedddf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"e2e","signature_id":"a9f1409cf050d25886bb76839830611031e806e434f180cb1affd1b0a192d798","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ff9c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dzgz54ntn7hc\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"name\": \"184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:43:16.227264536Z\",\n      \"endTime\": \"2026-09-23T14:02:20.066830362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ff9c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dzgz54ntn7hc\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"name\": \"184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:43:16.227264536Z\",\n      \"endTime\": \"2026-09-23T14:02:20.066830362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7102/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102726486111817728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7102,"pr_state":"closed","pr_sha":"d9646c1471c3fa0c1dafff1b68b88cec57beaa45","final_merged_sha":"d9646c1471c3fa0c1dafff1b68b88cec57beaa45","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T11:47:15Z","started_at":"2026-09-23T11:47:15Z","completed_at":"2026-09-23T13:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:10:07.687404345Z","timing_metadata_checked_at":"2026-09-23T14:10:07.687404345Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:10:07.690643334Z","region_metadata_checked_at":"2026-09-23T14:10:07.690643334Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7103,"pr_state":"closed","pr_sha":"174cb485dd2d6fb1ec807a48bb1bb2d984e4c005","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T11:38:49Z","started_at":"2026-09-23T11:38:49Z","completed_at":"2026-09-23T13:35:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:48:58.160482071Z","timing_metadata_checked_at":"2026-09-23T13:48:58.160482071Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:48:58.175036223Z","region_metadata_checked_at":"2026-09-23T13:48:58.175036223Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"106ef20d0f435c213e1519425e760f4c9723af37cc0b17014b0d9c3e3fbf51f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"e2e","signature_id":"312be1f9783180426404eca8136660afa7c8940a066aacb369c351024c0827a4","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"541fd6b77a464bfc038d951dadcd2d23614700a714c9d71ec79413000191e475","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"alert","signature_id":"b94154bec553ae4d7de90c235afb929dc5f8241123d4ff71acb4b324e3664bec","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T13:20:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6900864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-st2mg\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-st2mg (namespace arobit) on cluster ci01-j6900864-svc has averaged 124.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e184f90cd9d4a1e5055908b5b301b5ce9bbac921e2b0c1df5091b64f346f1ee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"alert","signature_id":"66ef3a438e94ca88c6470f62824d14213734daf6ab2142316c1d734ddeaa9c0e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:28:03Z\n  Ended:       2026-09-23T13:01:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6900864-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6900864-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7101/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102722381150687232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7101,"pr_state":"closed","pr_sha":"52944f9dea1b221bfcdc4f83c1349fea7ff49598","final_merged_sha":"52944f9dea1b221bfcdc4f83c1349fea7ff49598","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T11:30:56Z","started_at":"2026-09-23T11:30:56Z","completed_at":"2026-09-23T13:26:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:38:14.35999517Z","timing_metadata_checked_at":"2026-09-23T13:38:14.35999517Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:38:14.363564058Z","region_metadata_checked_at":"2026-09-23T13:38:14.363564058Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102722297386242048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"57066b220bb112a1dcac715da3b728883b34274f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T11:30:36Z","started_at":"2026-09-23T11:30:36Z","completed_at":"2026-09-23T13:40:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:48:58.007435382Z","timing_metadata_checked_at":"2026-09-23T13:48:58.007435382Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:48:58.013328962Z","region_metadata_checked_at":"2026-09-23T13:48:58.013328962Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T11:19:17Z","started_at":"2026-09-23T11:19:17Z","completed_at":"2026-09-23T13:19:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:28:04.761434518Z","timing_metadata_checked_at":"2026-09-23T13:28:04.761434518Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:28:04.765205806Z","region_metadata_checked_at":"2026-09-23T13:28:04.765205806Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6b6271afc67c6e8ee86afc9cf4b0788a0ccda653050ea6468a1ec1e9d667aca7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","occurred_at":"2026-09-23T11:19:17Z","failed_at":"alert","signature_id":"04cf9b464f041c4aa5f1606316891b1c2238b1191be6f182872d69d9ca55ffea","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:40:58Z\n  Ended:       2026-09-23T12:46:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1142016-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1142016-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T11:07:04Z","started_at":"2026-09-23T11:07:04Z","completed_at":"2026-09-23T13:08:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:17:40.999261135Z","timing_metadata_checked_at":"2026-09-23T13:17:40.999261135Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:17:41.002785522Z","region_metadata_checked_at":"2026-09-23T13:17:41.002785522Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a24a9058e17914bf6e7a75fc61011868b75db07b3fdad22bc056e572da02bbc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","occurred_at":"2026-09-23T11:07:04Z","failed_at":"alert","signature_id":"ecc22c0637cf2c040e23438995345ce727a3526729f1b45629b38665f2e2a492","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:03:55Z\n  Ended:       2026-09-23T12:37:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2532480-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2532480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102701497664933888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T10:07:57Z","started_at":"2026-09-23T10:07:57Z","completed_at":"2026-09-23T12:24:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:35:09.233554853Z","timing_metadata_checked_at":"2026-09-23T12:35:09.233554853Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:35:09.236909042Z","region_metadata_checked_at":"2026-09-23T12:35:09.236909042Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102701493466435584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7099,"pr_state":"closed","pr_sha":"2208f09500e712bff63abf4822601511464809c3","final_merged_sha":"2208f09500e712bff63abf4822601511464809c3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T10:07:56Z","started_at":"2026-09-23T10:07:56Z","completed_at":"2026-09-23T12:13:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:25:05.974862745Z","timing_metadata_checked_at":"2026-09-23T12:25:05.974862745Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:25:05.97737014Z","region_metadata_checked_at":"2026-09-23T12:25:05.97737014Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"ceedf6dfd596d82f101784fbf903aaa984937b18","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T10:05:03Z","started_at":"2026-09-23T10:05:03Z","completed_at":"2026-09-23T12:10:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:25:05.786905113Z","timing_metadata_checked_at":"2026-09-23T12:25:05.786905113Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:25:05.790614305Z","region_metadata_checked_at":"2026-09-23T12:25:05.790614305Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"68f489ba888e3d50dcca7a1d3b249eb56176cc6ca808f08bd1c7e5972bfae538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","failed_at":"e2e","signature_id":"80e228d9d0c46c7582a7eb683b187f1c62b13a5fdc9e873f52821f9b7629b50e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00190deb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00190deb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"a1867367778c664515dc8f00b2f77e472447bf8929f5a826301c9d56eb8acf3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","failed_at":"alert","signature_id":"695f2b58900eeb6ddc3f5c28514b819f16afd04fc4ec5e5122696e6f1efc6a9c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T11:26:05Z\n  Ended:       2026-09-23T11:37:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7059968-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7059968-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T09:32:18Z","started_at":"2026-09-23T09:32:18Z","completed_at":"2026-09-23T11:32:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:42:48.418910491Z","timing_metadata_checked_at":"2026-09-23T11:42:48.418910491Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:42:48.422422479Z","region_metadata_checked_at":"2026-09-23T11:42:48.422422479Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"672b3b5795cf93d757b5f7f36fe08af56dd457015a24b7919532a087a2f7659e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","occurred_at":"2026-09-23T09:32:18Z","failed_at":"alert","signature_id":"5be4b79fd707f0f902f0c6190406a62281b77b6ad2601f0cc1b46e4e850c930c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:53:08Z\n  Ended:       2026-09-23T10:59:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2996736-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2996736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"863a5e13aa5f5eec1b54764ecc9ee558bc55f41c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:30:30Z","started_at":"2026-09-23T09:30:30Z","completed_at":"2026-09-23T11:27:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:42:48.608523644Z","timing_metadata_checked_at":"2026-09-23T11:42:48.608523644Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:42:48.611398134Z","region_metadata_checked_at":"2026-09-23T11:42:48.611398134Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"04ba43610abb0045a6381bb56babd96c5fa3abffacc42050543339ad68597cf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","occurred_at":"2026-09-23T09:30:30Z","failed_at":"alert","signature_id":"a430843224ed1479b519c2f15a1c5fc4265a858e6f819daf935153b2c4d45bf9","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:52:54Z\n  Ended:       2026-09-23T10:58:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1762304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1762304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7077,"pr_state":"closed","pr_sha":"54cd641541a0e7abd44ce2fadd761db8cd8f95ca","final_merged_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:20:17Z","started_at":"2026-09-23T09:20:17Z","completed_at":"2026-09-23T11:20:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:32:45.096592993Z","timing_metadata_checked_at":"2026-09-23T11:32:45.096592993Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:32:45.099124384Z","region_metadata_checked_at":"2026-09-23T11:32:45.099124384Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"383375b56891fc6d6be7c2490ff8f3cc89c9259a61ade5cc1cb16a8dc49240eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","failed_at":"alert","signature_id":"acc0593d70344c6ce5881c51325fa15f3ec8bcecedf6ebedda37a38bcf852bf6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:25:13Z\n  Ended:       2026-09-23T10:52:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T10:25:14Z\n  Ended:       2026-09-23T10:54:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"885576e3a6fda561029ef56c863c71894b6435ef556c36dec718557fe5a1d1b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","failed_at":"e2e","signature_id":"7f41eb165d596d5dc7f2971efb22eb9df93680423e7b77c55b04839a62820eb7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"f8180cd6044fd44b58d22fb50e2a7d9429087dfe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:17:52Z","started_at":"2026-09-23T09:17:52Z","completed_at":"2026-09-23T11:24:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:32:44.972534714Z","timing_metadata_checked_at":"2026-09-23T11:32:44.972534714Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:32:44.975882903Z","region_metadata_checked_at":"2026-09-23T11:32:44.975882903Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"21796c2413243b95cf2304b1af8be05d892462a25208ff8862c31e530ce07077","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","occurred_at":"2026-09-23T09:17:52Z","failed_at":"alert","signature_id":"168b46660d0ed6f4bf2e6d271e550940fff7e50d9ae0544d3d77fc3aaa3c674c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:47:31Z\n  Ended:       2026-09-23T10:53:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6677120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6677120-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102679333091938304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7070,"pr_state":"closed","pr_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","final_merged_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T08:39:52Z","started_at":"2026-09-23T08:39:52Z","completed_at":"2026-09-23T10:46:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:01:41.898232855Z","timing_metadata_checked_at":"2026-09-23T11:01:41.898232855Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:01:41.902008642Z","region_metadata_checked_at":"2026-09-23T11:01:41.902008642Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T08:02:37Z","started_at":"2026-09-23T08:02:37Z","completed_at":"2026-09-23T10:11:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T10:30:38.200722845Z","timing_metadata_checked_at":"2026-09-23T10:30:38.200722845Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T10:30:38.203847735Z","region_metadata_checked_at":"2026-09-23T10:30:38.203847735Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"80918aeb625d32fdb0e8534a588ebd2d48f0c1da28fe81b6d18ca6c9ea3b1d01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","occurred_at":"2026-09-23T08:02:37Z","failed_at":"alert","signature_id":"ef8ed7f81b6e444cb9f47b23d2f051a1e779787724507e05e79e426102d41cd3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T09:00:54Z\n  Ended:       2026-09-23T09:34:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4854400-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4854400-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T08:02:34Z","started_at":"2026-09-23T08:02:34Z","completed_at":"2026-09-23T10:02:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T10:20:29.53596881Z","timing_metadata_checked_at":"2026-09-23T10:20:29.53596881Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T10:20:29.539446698Z","region_metadata_checked_at":"2026-09-23T10:20:29.539446698Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0293d84de252cc10fa4aba8b36b0124251c3aaff37c8e07cdc46188972b5f89d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","occurred_at":"2026-09-23T08:02:34Z","failed_at":"alert","signature_id":"6a553ba910a4f3a893c5726596a17745a39160fc0a64f583a6cc5c05a5a4ea9e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T09:48:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6558336-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j66dp\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-j66dp (namespace arobit) on cluster ci01-j6558336-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102659427256504320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T07:20:47Z","started_at":"2026-09-23T07:20:47Z","completed_at":"2026-09-23T07:30:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:36:51.480167516Z","timing_metadata_checked_at":"2026-09-23T07:36:51.480167516Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T07:36:51.482770407Z","region_metadata_checked_at":"2026-09-23T08:40:09.114907986Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"45cadb2e34fb786fe9639fb0838e0d13c322eaa13adc1aa7f611ad31e0a8a4f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102659427256504320","occurred_at":"2026-09-23T07:20:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102655845220225024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4878,"pr_state":"closed","pr_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","final_merged_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T07:06:33Z","started_at":"2026-09-23T07:06:33Z","completed_at":"2026-09-23T09:03:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T09:17:56.890736447Z","timing_metadata_checked_at":"2026-09-23T09:17:56.890736447Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T09:17:56.893966036Z","region_metadata_checked_at":"2026-09-23T09:17:56.893966036Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"1769127263098290ba69f6f4e9bc0696f1c66e33","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:50:00Z","started_at":"2026-09-23T06:50:00Z","completed_at":"2026-09-23T08:55:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:57.542614074Z","timing_metadata_checked_at":"2026-09-23T08:56:57.542614074Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:57.54614706Z","region_metadata_checked_at":"2026-09-23T08:56:57.54614706Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c1e6ae9e98fc46a47884a91a6f65821208f2208f21b1b8fa8fd0373da726bcf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","occurred_at":"2026-09-23T06:50:00Z","failed_at":"alert","signature_id":"01cbd1d336e53f82a367140655d108e641e536db1311bf518e94e91e812f8a75","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:33:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j5147264-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j5147264-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"open","pr_sha":"b6ef3476b2950b7c03a8b9bbd77712889cebd2df","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:49:08Z","started_at":"2026-09-23T06:49:08Z","completed_at":"2026-09-23T09:00:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T09:17:56.908758485Z","timing_metadata_checked_at":"2026-09-23T09:17:56.908758485Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T09:17:56.911433976Z","region_metadata_checked_at":"2026-09-23T09:17:56.911433976Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"313a8e1451063175a424d0d816daef49504688162aa45df25cdcc656d0471bcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"alert","signature_id":"61fa061764ad73e8317dc73f0ca153b35f59fa073234e6998c92f96b627530c7","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:47:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3684224-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z8dk5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z8dk5 (namespace arobit) on cluster ci01-j3684224-svc has averaged 125.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"52d9bb0ef79dcf0073fa404bc89036b655c5d0aaf88ae71236091f8ddf4b2596","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"e2e","signature_id":"a02ab509af0dce2add587b938aa804dece2558ca7621ffc208e009f99dae4089","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","failure_pattern":"expected all guest nodes to be unschedulable, schedulable=\u003cnode\u003e, \u003cnode\u003e; nodes: \u003cnode\u003e ready=true unschedulable=true created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=true unschedulable=false created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=false","search_query":"expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane"}},{"row_id":"e1424293e6ef937d8f58a7b1246d19e995f47c884731c8311815c0385ee46e3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"alert","signature_id":"24389b82f6419bf8544e5612ab91556f9033a43c11c58c8a75f9eddd0d3cb5d2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:11:35Z\n  Ended:       2026-09-23T08:22:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3684224-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3684224-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7099,"pr_state":"closed","pr_sha":"2208f09500e712bff63abf4822601511464809c3","final_merged_sha":"2208f09500e712bff63abf4822601511464809c3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T06:46:50Z","started_at":"2026-09-23T06:46:50Z","completed_at":"2026-09-23T08:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:58.215393187Z","timing_metadata_checked_at":"2026-09-23T08:56:58.215393187Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:58.218456976Z","region_metadata_checked_at":"2026-09-23T08:56:58.218456976Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"294ca3dfd83b32cf8128240e53199485e5b32182c9d77acf19b518b1387f1c93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","failed_at":"alert","signature_id":"7b5a0fe8e7d62a0661702614b741d3d10ab9bf69cb3d6aad1f5d3a0807f9d130","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:39:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j3407488-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j3407488-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}},{"row_id":"33cf1cfbd556ae52ff52d99d61dcedc1b01d5be2c1e8939a0b304abfc63d5c57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","failed_at":"alert","signature_id":"7c059f6930e4e70e01cf60cdc5a209a41e1d36e3c5cdc0a3889876307b50e19b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:08:37Z\n  Ended:       2026-09-23T08:22:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3407488-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j3407488-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"57066b220bb112a1dcac715da3b728883b34274f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:44:17Z","started_at":"2026-09-23T06:44:17Z","completed_at":"2026-09-23T08:53:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:57.863225578Z","timing_metadata_checked_at":"2026-09-23T08:56:57.863225578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:57.865784969Z","region_metadata_checked_at":"2026-09-23T08:56:57.865784969Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"6e7f9e6dca9ada1bde642f56305a573d151deac2df7e1fdedc4cee00a17e8fe1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","failed_at":"alert","signature_id":"9b7c4ba8bc11e40e7d6443d7bd87ac580a2605e35dba3d0581d0e43dcef62d3d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:11:20Z\n  Ended:       2026-09-23T08:18:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5172864-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5172864-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"ce8f0e173cb61e12360c71d260495a0a6f1c2a8281fe73d298d184e3ffb4eb87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","failed_at":"alert","signature_id":"398824eec9025702d224d5a9ba36e64c2703dfdc5e00ba0eb6afbc886a9d202d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:39:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5172864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j4msz\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-j4msz (namespace arobit) on cluster ci01-j5172864-svc has averaged 124.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7051/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102639009233637376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7051,"pr_state":"open","pr_sha":"e7b5fbaebd5615c4c4ed6f61605b5b8811a4947c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T05:59:38Z","started_at":"2026-09-23T05:59:38Z","completed_at":"2026-09-23T08:10:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:18:41.894413233Z","timing_metadata_checked_at":"2026-09-23T08:18:41.894413233Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:18:41.900036514Z","region_metadata_checked_at":"2026-09-23T08:18:41.900036514Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T05:02:00Z","started_at":"2026-09-23T05:02:00Z","completed_at":"2026-09-23T07:13:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:26:35.782975741Z","timing_metadata_checked_at":"2026-09-23T07:26:35.782975741Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T07:26:35.78648243Z","region_metadata_checked_at":"2026-09-23T07:26:35.78648243Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"86057d21266144a0386954db6763cf7e49579e2c47a6c299ae788a922b6f3b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","occurred_at":"2026-09-23T05:02:00Z","failed_at":"alert","signature_id":"fec44601492cd080815bfd1f2fda9d583e0a942b12d71ab0789e80aa75b0494f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T06:11:33Z\n  Ended:       2026-09-23T06:20:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3290112-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y\", pod=\"community-operators-catalog-5f9d56fbbf-bvthw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y/community-operators-catalog-5f9d56fbbf-bvthw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T05:01:42Z","started_at":"2026-09-23T05:01:42Z","completed_at":"2026-09-23T07:17:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:26:35.907242631Z","timing_metadata_checked_at":"2026-09-23T07:26:35.907242631Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T07:26:35.910308421Z","region_metadata_checked_at":"2026-09-23T07:26:35.910308421Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"38f088f8d1d9f1694bec5e46d0e1b3eae1afd9b68c8e98a1bc85be6786d5f064","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","occurred_at":"2026-09-23T05:01:42Z","failed_at":"alert","signature_id":"e61751d58352ab491f0598c208a923b5d0b7b5b03f412d966b5363851718b33b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T06:43:51Z\n  Ended:       2026-09-23T06:49:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1547520-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1547520-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102620674894860288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T04:46:48Z","started_at":"2026-09-23T04:46:48Z","completed_at":"2026-09-23T04:55:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T05:11:21.468834083Z","timing_metadata_checked_at":"2026-09-23T05:11:21.468834083Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T05:11:21.472427671Z","region_metadata_checked_at":"2026-09-23T06:12:21.316214356Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"db9ef5b98ba20063f2855cc462d10730ad69b3e2601efcf25eb480fd4725b169","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102620674894860288","occurred_at":"2026-09-23T04:46:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7098/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579983321927680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7098,"pr_state":"closed","pr_sha":"f863469794c6e0ad6c0a4d8c8fd4f2a8051647db","final_merged_sha":"f863469794c6e0ad6c0a4d8c8fd4f2a8051647db","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T02:05:06Z","started_at":"2026-09-23T02:05:06Z","completed_at":"2026-09-23T04:11:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:17:58.158395822Z","timing_metadata_checked_at":"2026-09-23T04:17:58.158395822Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:17:58.161533612Z","region_metadata_checked_at":"2026-09-23T04:17:58.161533612Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T02:01:18Z","started_at":"2026-09-23T02:01:18Z","completed_at":"2026-09-23T04:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:07:14.580204121Z","timing_metadata_checked_at":"2026-09-23T04:07:14.580204121Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:07:14.582789412Z","region_metadata_checked_at":"2026-09-23T04:07:14.582789412Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"0eb1ca3df25acb4111a6e19897c009e53a9ba98abc3b19f34fc90de0cfbe6cf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"b52a16123591cbdade9825eddb9e37de34eaab046973c4b8e648b0ae88335846","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T03:16:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 250ms over the past 30 minutes (cluster ci01-j3482624-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T03:29:12Z\n  Ended:       2026-09-23T03:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3482624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9dcd032efeed8c80f0a430d4e4579b817e20e336e340f47fbff13e9c5a927e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T03:10:35Z\n  Ended:       2026-09-23T03:20:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3482624-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"a8d424a7f632829120e0f9b5b5977b2c2480a331622f269b3e60a103470b27bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"b20cadb846b55234efc43634cdf321fda9be13c6186765f74dd9bcf81c873e27","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T03:37:03Z\n  Ended:       2026-09-23T03:47:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j3482624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7089/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579123015651328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7089,"pr_state":"closed","pr_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","final_merged_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T02:01:11Z","started_at":"2026-09-23T02:01:11Z","completed_at":"2026-09-23T04:08:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:17:58.031253737Z","timing_metadata_checked_at":"2026-09-23T04:17:58.031253737Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:17:58.034839626Z","region_metadata_checked_at":"2026-09-23T04:17:58.034839626Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T01:27:26Z","started_at":"2026-09-23T01:27:26Z","completed_at":"2026-09-23T03:17:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T03:25:11.165946146Z","timing_metadata_checked_at":"2026-09-23T03:25:11.165946146Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T03:25:11.169534334Z","region_metadata_checked_at":"2026-09-23T03:25:11.169534334Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"084d748eb8dde746c988f9a89fea25877605d507aad827cb31079b3f1adf251e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","occurred_at":"2026-09-23T01:27:26Z","failed_at":"e2e","signature_id":"75beab8f572182a387939937ea4f447a02998cc8ddb2ad7397436d6df460ffc0","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T01:02:20Z","started_at":"2026-09-23T01:02:20Z","completed_at":"2026-09-23T02:21:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:34:36.179192523Z","timing_metadata_checked_at":"2026-09-23T02:34:36.179192523Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:34:36.182802611Z","region_metadata_checked_at":"2026-09-23T02:34:36.182802611Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"97a00ad8ca6ddca5e4e12c96f8c3a591af8cbd5c5474f9ae07e31cffe0e93ee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"cd2acdca3a5c4271f034e124fe4ffb19ca10970d45a2e64d9db71f66d0c505d1","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-ch8gg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-ch8gg has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9ptjq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-9ptjq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-kw5rf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-kw5rf has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cxgg8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-cxgg8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"c3227c73c9e918743470589e845ca9073ba8bb9cb27f129d4b5eebc2a1c0db91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"provision","signature_id":"f020f9342f43cace837cc83b8952affa433a8c7410f5746e59533d769ed278b9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T01:45:47.799Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:48.667Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:49.334Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:55:54.330Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:55:54.331Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:56:54.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:55.777Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.630Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T01:56:57.690Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:57.691Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.299Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.862Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:56:59.423Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:57:00.012Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.364Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.910Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:57:01.546Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T02:07:04.175Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:07:04.175Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:08:04.175Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:05.466Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.382Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:07.843Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:08.593Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.447Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.853Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:10.521Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:12.134Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:15:47.989Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T02:15:47.990Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"eb5a7eb26294cc47e37765b4b6c6d557f498ff7a6d3497d50563ce0fd68440f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"80bbd3a02e54b8fc7971960324a0123e49b270a5d705be23d017d8e1a4b2f32e","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"ec99c812f36f32216c370a3ebe0c6c035e70c262b41bc386f59c11dce901f8a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"0bff05a4b68cd34010dac9320f26f4a4adb189546a09d7a89930f18e9ffb6349","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0769280-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j0769280-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564169285898240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7064,"pr_state":"closed","pr_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","final_merged_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T01:02:16Z","started_at":"2026-09-23T01:02:16Z","completed_at":"2026-09-23T03:01:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T03:14:58.015236856Z","timing_metadata_checked_at":"2026-09-23T03:14:58.015236856Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T03:14:58.018693443Z","region_metadata_checked_at":"2026-09-23T03:14:58.018693443Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7085/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102561003798532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7085,"pr_state":"closed","pr_sha":"b8e74897ad36b238125a1aefbaf68e7eae50c7d9","final_merged_sha":"b8e74897ad36b238125a1aefbaf68e7eae50c7d9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T00:49:41Z","started_at":"2026-09-23T00:49:41Z","completed_at":"2026-09-23T02:39:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:44:49.484343071Z","timing_metadata_checked_at":"2026-09-23T02:44:49.484343071Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:44:49.488061359Z","region_metadata_checked_at":"2026-09-23T02:44:49.488061359Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102549176721084416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T00:02:31Z","started_at":"2026-09-23T00:02:31Z","completed_at":"2026-09-23T02:06:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:13:15.093950874Z","timing_metadata_checked_at":"2026-09-23T02:13:15.093950874Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:13:15.097662262Z","region_metadata_checked_at":"2026-09-23T02:13:15.097662262Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102809541325361152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T17:17:16Z","started_at":"2026-09-23T17:17:16Z","completed_at":"2026-09-23T19:46:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:54:41.146446414Z","timing_metadata_checked_at":"2026-09-23T19:54:41.146446414Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:54:41.150371502Z","region_metadata_checked_at":"2026-09-23T19:54:41.150371502Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102710804108283904","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T10:44:56Z","started_at":"2026-09-23T10:44:56Z","completed_at":"2026-09-23T13:02:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:17:41.021666456Z","timing_metadata_checked_at":"2026-09-23T13:17:41.021666456Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:17:41.024578045Z","region_metadata_checked_at":"2026-09-23T13:17:41.024578045Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2910510a3db6da974f8261ebb971c2bfb6eb00cd0121e2bf4064b4b254e1a564","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102710804108283904","occurred_at":"2026-09-23T10:44:56Z","failed_at":"e2e","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102650382160236544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T06:44:51Z","started_at":"2026-09-23T06:44:51Z","completed_at":"2026-09-23T08:57:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T09:17:56.926574724Z","timing_metadata_checked_at":"2026-09-23T09:17:56.926574724Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T09:17:56.929173415Z","region_metadata_checked_at":"2026-09-23T09:17:56.929173415Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c0060f3c0e73fe954cf1f8022a89bb2b595ae72fb96eea23ce0df8e8277be63a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102650382160236544","occurred_at":"2026-09-23T06:44:51Z","failed_at":"e2e","signature_id":"0513c9dba1d81c84b15ffbc8b39cea7d4f8ffd6ade45b91dfec359ec896f33e5","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6010\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["09d6899fcee106147461837dfde3776fb25ff9f2af5b0d715a24d5e92bbbf532"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T23:00:13Z","started_at":"2026-09-23T23:00:13Z","completed_at":"2026-09-24T00:46:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:51:35.682699068Z","timing_metadata_checked_at":"2026-09-24T00:51:35.682699068Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:51:35.686121457Z","region_metadata_checked_at":"2026-09-24T00:51:35.686121457Z"},"failed_at":["e2e"],"failed_test_count":19,"occurrences":[{"row_id":"07c32a800c2763c4c3b4dd3e34850a961cbf243ae1d71a75bd0be241f706395c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"5e96aa58c18568a25ab9acb5502820d47261ecacba6ffd475cef3d142037f291","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc2d80\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rdvqqgkbcqkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n      \"name\": \"a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:51:44.432225393Z\",\n      \"endTime\": \"2026-09-24T00:10:47.655792485Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bc2d80\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rdvqqgkbcqkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n      \"name\": \"a8f31d96-4bf3-4b48-a764-fe35d5af46db\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:51:44.432225393Z\",\n      \"endTime\": \"2026-09-24T00:10:47.655792485Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"32c1fd05dbc20eb4cc7dd75fc471b87340a055863fbf3178551edb97e55a4152","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"a6b4d2a29f145149ab937c5195b130875034ebc08e0235a73002c81ef5d5096f","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fe4a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9tsbffh9csll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tsbffh9csll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fe4a0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9tsbffh9csll\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tsbffh9csll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"331d3e169e5f66ee0e9d388feb405c40744f21598caae67eac0c0c66b62ff85d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"2c73a06cdcacfc08c5ae0807302796c3e3154da70a86dd6ed16c84098e26ab85","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cca500\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-4fchkp\" in resourcegroup=\"rg-np-skip-minor-4fchkp-d9wq5jxbr4vg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-4fchkp-d9wq5jxbr4vg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-4fchkp/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cca500\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-4fchkp\" in resourcegroup=\"rg-np-skip-minor-4fchkp-d9wq5jxbr4vg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-4fchkp-d9wq5jxbr4vg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-4fchkp/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"36c325356f8efcbaa52b66e99eb643162f9b8aa0fc8f8d8928298aaceda6d8ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"696c58cbd3aa8d300062df4a47583e3f809311c672a43e3da76b7164514ff28c","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96700\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nfm8pxmkddmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nfm8pxmkddmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f96700\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nfm8pxmkddmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nfm8pxmkddmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"37d5748e9d2722d9bac0b3475a724500ef1c4d4bbd036e348b85f1ea91c66889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"ccd3082ac0ec3ee7f5872b88cabf71e9eaf1a7b9a9598367688dc3bb14866a3a","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd6f60\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7225cv\" in resourcegroup=\"rg-np-version-upgrade-7225cv-t79q2jkmqh7t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7225cv-t79q2jkmqh7t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7225cv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd6f60\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7225cv\" in resourcegroup=\"rg-np-version-upgrade-7225cv-t79q2jkmqh7t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7225cv-t79q2jkmqh7t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7225cv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"510b8e493ea92572817dd34a7398c288a657850a731cfcd7d4d8b9d8654c3af8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"2f40037715e35fc92a35242fe15cfd48e9dba77dbef0411ab7466b2e286500e4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000587ac0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f9strbqhzqsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f9strbqhzqsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5162786bc2c2a8fbed013dcaed75201af7b9cadcb2cb9c27dd8832e8572db6c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"9099db8102021ee82d4f7fa885ff9d6c40c7a777e05056ac01595d8cbe6b71e0","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013365e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-87rcp5g2p96w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-87rcp5g2p96w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013365e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-87rcp5g2p96w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-87rcp5g2p96w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6f9d7066bb98f225e4b1bf00161b70bae96e8757fc8ac78543d7f86830727c88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"0b19163b0950f2bc9dac2fa7bbe2fe9b7e8b4eddb5161aef3f7f5ec4427662e9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002a7cd50\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-lqwwzcqjkl4d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002a7cd50\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-lqwwzcqjkl4d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"75881a418f7533ec47b77112249426e77459695e4c2c6b1013cdbbe6c07f3a6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"a15265dd30984b38cf2e97f6a7e9fa79ca18ae3cedd2d533ed100ef489695a0f","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f5220\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gp6h5d2jnk5d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-gp6h5d2jnk5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f5220\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-gp6h5d2jnk5d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-gp6h5d2jnk5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"81845bcb31a7fa73754298e758e3de3b8a4ba984703ec247a634e1944da9eada","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"9fb7620a84e0a14940bbc581680a78cbf6f7134a9005afd5f2451b87d1987486","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001181280\u003e: \n    failed to create NodePool np-lqb7d8: failed starting nodepool creation \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\" in resourcegroup=\"rg-nightly-4-20-9g5dwcgvzm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9g5dwcgvzm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-lqb7d8/nodePools/np-lqb7d8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001181280\u003e: \n    failed to create NodePool np-lqb7d8: failed starting nodepool creation \"np-lqb7d8\" for cluster \"cluster-nightly-4-20-lqb7d8\" in resourcegroup=\"rg-nightly-4-20-9g5dwcgvzm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9g5dwcgvzm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-lqb7d8/nodePools/np-lqb7d8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8dff730714f53a9b974dfa52738163b2e0b67f0c0bc0befbb34129689434fa23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"8b972b568eeb592cb7c4e3ea2dd32eb95af26dd5cf16683b957140e9cded71c1","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000618ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5n2hfst68hj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b5n2hfst68hj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000618ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5n2hfst68hj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b5n2hfst68hj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9389ef0c0452bbdf286815aec93a1ec8c81a9bd453cc6bc395189835337bdda5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"4beec4b23a49b2c4afc633ee4de74ca68b43b8c0a169e74f6e8c71ff427a668d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001570180\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9vczdt9qztzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001570180\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9vczdt9qztzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a21ffac1864fe7118e8f3d55c0298022ac5954314013260a344db67310188a58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"3532da433aa865198e7af69b2d67f9f685db4cc1befc31928db022dff5c8ba6f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002564810\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cr6zxd6vcc7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002564810\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cr6zxd6vcc7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bd53230c3b6bdc6bc29e910b2677906fe9dfbd0e59fff741c5015926d109c878","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"e5a3278c01c0abce683e00630bbc784dbd610e4e57d5f23368cb17f09c05ad78","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104b000\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-4t6q2594ttzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-4t6q2594ttzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00104b000\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-4t6q2594ttzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-4t6q2594ttzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bf12a4ab02eb3e71390f1b5cb5c5542ee8fdb59cf9c5cd9cbeaac1384886e82b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"af6920bd4dcef413e75f2c2ee8ed9b6ea9da89a5c3cbc85c4911bd464e6f591d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mx2sdx5hq5cq/cluster-nightly-5-1-r4tn78 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001788aa0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-r4tn78: failed waiting for cluster=\"cluster-nightly-5-1-r4tn78\" in resourcegroup=\"rg-nightly-5-1-mx2sdx5hq5cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n      \"name\": \"9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:28:28.748187971Z\",\n      \"endTime\": \"2026-09-23T23:57:28.896085734Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-mx2sdx5hq5cq/cluster-nightly-5-1-r4tn78 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001788aa0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-r4tn78: failed waiting for cluster=\"cluster-nightly-5-1-r4tn78\" in resourcegroup=\"rg-nightly-5-1-mx2sdx5hq5cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n      \"name\": \"9e1c75fb-14a3-4cb5-b5d0-9a40f34a6079\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:28:28.748187971Z\",\n      \"endTime\": \"2026-09-23T23:57:28.896085734Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c51470cb51ff01224cce9780d22b2b435b4730eb5ccaaa814e46fc1c580c8f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"80af8612530648ca50b987b6f9cdbcfec5e5733a4eb73b613f4f2cca40440dda","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001010900\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-2lsdzccng668/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001010900\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-2lsdzccng668/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c90d66d04077ea742d8b67a0b1d4dd5cac1babe20a80a55be52db8b3e52c5a46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"c0473fca37809ede533480e60205fed102ad16eccbd4d3d6441046e31b66bf9d","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013ce030\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dcrp5phx2tdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013ce030\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-dcrp5phx2tdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e1e43a269be021e03afe6e4a788d4bb5ff3257fc7fedd2c01a7e15772bed6358","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"57f75e9006bbdcb6503c784e4aae06190b496bc4d2463173bd66d696b13fcfb8","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ed00\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zj2nbd\" resourcegroup=\"rg-np-dg-minor-zj2nbd-nbv87cpm7mgj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zj2nbd-nbv87cpm7mgj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zj2nbd/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00121ed00\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zj2nbd\" resourcegroup=\"rg-np-dg-minor-zj2nbd-nbv87cpm7mgj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zj2nbd-nbv87cpm7mgj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zj2nbd/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fc12d2d7db1ae86e5178c405a56cf46e10f714a721459d412ed24bfe9c5e805d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2102895842133282816","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"b7d9ce39ea6386c93e86493f0c036b5ac8e6406a7e67ba9a7045486fddb68882","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cbd410\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-23-031212, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cbd410\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-23-031212, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":19,"failure_pattern_ids":["311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":19,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T02:00:41Z","started_at":"2026-09-23T02:00:41Z","completed_at":"2026-09-23T03:58:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:07:14.740977759Z","timing_metadata_checked_at":"2026-09-23T04:07:14.740977759Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:07:14.743908949Z","region_metadata_checked_at":"2026-09-23T04:07:14.743908949Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"2aaed42f233b29555ae3776af4e4e16031bc55f7b50cba058a227a7c407b921a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"f40ba7ba94d5d5cc9b8228f74ef22fcc6c8cf1de075f86ae5b726e2a4ba76bcb","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c08920\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-xlkqvsjbnblh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-xlkqvsjbnblh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c08920\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-xlkqvsjbnblh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-xlkqvsjbnblh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3ec85b67f8eede7095471674a795d3c6addffe6ede8e6283e918ba1f8559b6f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"950be739fb33f7b6adecbfc5254b9a9b8d54463fd0cd14ad47b8f998fd5afdc9","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e91540\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-6r7w984km7pq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-6r7w984km7pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e91540\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-6r7w984km7pq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-6r7w984km7pq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"41aee21ff4adf90a6baa582d2cdc2b06c6209168b1bb07f251eb680f62891b93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"60163a0f600eba5c06ff2c878eb2a0092d005dbebf2c0e82fe91feb85ffe2ec7","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9ede0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zs28fdvptj4b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zs28fdvptj4b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9ede0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zs28fdvptj4b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zs28fdvptj4b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4534ffdf471786db1b2c31aebd324d53fec6690d4cd9fa5a4e86173c24f75699","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"ef884a233cc88172af6105a108535d3de27389d5003c528f736c27bc0329a063","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001013f60\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-hn98pgcl2759\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-hn98pgcl2759/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001013f60\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-hn98pgcl2759\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-hn98pgcl2759/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"53a91a47abe81a87064c63a71de9b7167ce7b8a0fdb69c230ca8dbde16c6e6b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"dcdb32a276b85d5762c8f9a4390b250e8e8c154b174fce2166a1284975070064","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block\nExpected\n    \u003c[]uint8 | len:6010, cap:6012\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","failure_pattern":"cluster CA data must contain exactly one PEM block","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:202]: cluster CA data must contain exactly one PEM block"}},{"row_id":"721340579dfcc2fe1e75f83531b33ff7d9ff26662461adfce2e85978aaf569d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"3c47413f0338789d70091286ea56544fcdee01d67131f0f7851a0b97474d7105","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fd38c0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-d8vr2mxrtk2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fd38c0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-d8vr2mxrtk2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7f624df8d1118e3c7ae9c1c1f76db674812a2db4379df31d05c9bcea6e304bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"b2a0aa0f0d5efc781c39edb1e340969ee91b48c4052331e8dd71b8357572e91b","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a27d40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6t9mhgvsbfj6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-6t9mhgvsbfj6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a27d40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6t9mhgvsbfj6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-6t9mhgvsbfj6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"940d0286c84f0a9370fef36a3cb6e38643f08b9adc9ecad81f511acc66602a63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"a1b9cf6305c1912800fbad4a1b18908456f24a8e3cd6243102d4426c2fb0976f","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfede0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9ssdcjgnxb8x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9ssdcjgnxb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfede0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9ssdcjgnxb8x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9ssdcjgnxb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e3a240cd040507c4af299b4f874285530f407951f55469664c78cf25b173df59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102578870526939136","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"300e775547bf8a76fdcf7fc914a06bfeb3d60330ccc0c5023680429a57300fc7","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002b8f878\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v52dlnd99fqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\",\n      \"name\": \"8dff53d4-fb68-4baa-8655-9059740a3aee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T02:19:20.475025137Z\",\n      \"endTime\": \"2026-09-23T02:38:20.795241715Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002b8f878\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v52dlnd99fqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dff53d4-fb68-4baa-8655-9059740a3aee\",\n      \"name\": \"8dff53d4-fb68-4baa-8655-9059740a3aee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T02:19:20.475025137Z\",\n      \"endTime\": \"2026-09-23T02:38:20.795241715Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["1c0eb3626837a0d57ffaef6abf34daac6ebcb26dfffc58a3917d8718d8239c35","311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T23:34:29Z","started_at":"2026-09-23T23:34:29Z","completed_at":"2026-09-24T00:12:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:19:49.122475498Z","timing_metadata_checked_at":"2026-09-24T00:19:49.122475498Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:19:49.126149785Z","region_metadata_checked_at":"2026-09-24T00:19:49.126149785Z"},"failed_at":["e2e"],"failed_test_count":58,"occurrences":[{"row_id":"00ff0071470574c11372a143fb2066921e2b67073a3296d63461cbd45b53e9ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"ad868390107afef487eb4050fa0c3e0b12550c6ce4d10603b345f5c954ed7fb6","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012078e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-lsql8k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-lsql8k/operationStatuses/08584113986461824979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9lbzntwzvchs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012078e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-lsql8k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-lsql8k/operationStatuses/08584113986461824979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9lbzntwzvchs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"02fcb639b7c5ace34ef162467d0b2d6957bc070167f5f8ec7eb2ac94521131ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"848a835912eb35c98e5793402aff5be6e85e6150ba7567ff6939266d31765a3c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ead40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-g6c2l8pqzc65\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-g6c2l8pqzc65/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ead40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-g6c2l8pqzc65\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-g6c2l8pqzc65/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"03b0a2510d5fb1649c5d5cfd4135ed0d84f0d71dcb27b59d85dfd8a38e35aa75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"baab246acfa194162e561c3c3cb6c1341eae11f128345f24470a2c72bd2f4aee","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-b77rqs\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2a3c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-b77rqs: failed starting cluster creation \"cluster-zstream-4-22-b77rqs\" in resourcegroup=\"rg-zstream-upgrade-4-22-x86pcxvscv78\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-x86pcxvscv78/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-b77rqs\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-b77rqs\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2a3c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-b77rqs: failed starting cluster creation \"cluster-zstream-4-22-b77rqs\" in resourcegroup=\"rg-zstream-upgrade-4-22-x86pcxvscv78\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-x86pcxvscv78/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-b77rqs\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1dd66304ace2ca917d58d7140bbdc47184c1644dad70dcf0c09f13e6c232a1d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"d7884279496d93385c3451a48331a1291d1cbd9cd69dab41dcdaacfdd026285a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a280\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zsl2zqg8d25s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zsl2zqg8d25s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a280\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-zsl2zqg8d25s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zsl2zqg8d25s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1e2a0f93ea4d3cc8ec49bd07a1f72d9c0e9312e2faee7f5d98c212e67fb04bec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"4c9e4e595392b76de5e57905da765e092ea4b53fb4e4a74a9a478b70b2a59a09","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa20e0\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ng56lz7dl9cb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ng56lz7dl9cb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa20e0\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ng56lz7dl9cb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ng56lz7dl9cb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1f41768a423a3e194c6d342cf1110bab8de19a9ce9778b15feb69f856edd4828","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"44477526f3f5c967017a65f3a9a523c776dd67b3ac4e6dd0403372636e3b1343","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8c120\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qx726fw4j5rl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-qx726fw4j5rl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8c120\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qx726fw4j5rl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-qx726fw4j5rl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"21cc5ee549ccc6307d2258462a3966bef8c930c6a3d92617b4da9875242ab940","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5783e4c8b0c913d9ac1cfb0f45eb24edd6329f5cb3aaa18343f050038c5e6d71","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wc8jdx\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009e0640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-wc8jdx: failed starting cluster creation \"cluster-zstream-4-20-wc8jdx\" in resourcegroup=\"rg-zstream-upgrade-4-20-9dr7b9qsqclg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-9dr7b9qsqclg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-wc8jdx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-wc8jdx\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009e0640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-wc8jdx: failed starting cluster creation \"cluster-zstream-4-20-wc8jdx\" in resourcegroup=\"rg-zstream-upgrade-4-20-9dr7b9qsqclg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-9dr7b9qsqclg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-wc8jdx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"21d28e126ee73e7124d58ac2409685b452c92109d71b2d1ec999557b918caed4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"ad8d9c518342d140737ad5d4a5fabe301705900959b3e86d1f3308ce70e5f962","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a092e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listxdjmk6-rp7t27\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listxdjmk6-rp7t27/operationStatuses/08584113987138459688\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-g8gfj6vr8skb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rlufqgosrjhbk\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a092e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listxdjmk6-rp7t27\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listxdjmk6-rp7t27/operationStatuses/08584113987138459688\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-g8gfj6vr8skb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rlufqgosrjhbk\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"243bf66e747d61264bd019f6223f06d5651df7b0a37d5c93b6c4165217238dba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"fc7baf411234e091570207b597ac8f8d793f9adeefa4cf31f1037c197fa10337","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000619cc0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-fzl4fk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-fzl4fk/operationStatuses/08584113987139523817\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-brhj6q4dslz6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000619cc0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-fzl4fk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-fzl4fk/operationStatuses/08584113987139523817\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-brhj6q4dslz6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2a4e56c25dea49df50ff00e3432cfa80b799a1feda0027ee84331410612ae75f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"7980065f3e3c71ff5248c023d7dea8b862c74b6f0fc9d85d1b0dc6232900a78d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-97lxrpzvwp8t/cluster-candidate-4-21-mq5ksz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000982100\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mq5ksz: failed starting cluster creation \"cluster-candidate-4-21-mq5ksz\" in resourcegroup=\"rg-candidate-4-21-97lxrpzvwp8t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-97lxrpzvwp8t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-mq5ksz\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-97lxrpzvwp8t/cluster-candidate-4-21-mq5ksz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000982100\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-mq5ksz: failed starting cluster creation \"cluster-candidate-4-21-mq5ksz\" in resourcegroup=\"rg-candidate-4-21-97lxrpzvwp8t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-97lxrpzvwp8t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-mq5ksz\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2c9f3a7fa17899df1ea65efd90c217ca5678bbe4e975628c5a009a6d1dd0ed4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"6b6231f6a741045d0d4adc8542106388a4942ccd90466630d8b1d4ae0197568e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d6fa0\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n662f9vjlzcr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-n662f9vjlzcr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d6fa0\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n662f9vjlzcr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-n662f9vjlzcr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2d59d063425a11550334b414ae2f53b4a6a8dd39e851dd03efeb32b4d48b5de4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"147bc66a9ffa2ef0eb84aeffac927813f9dbc0de655e0de415f62f1152f420b5","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e4b80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-42bbcr-vkf8rz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-42bbcr-vkf8rz/operationStatuses/08584113986227757596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-42bbcr-2rk6t6mlf2lm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pgend26hbad42\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001e4b80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-42bbcr-vkf8rz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-42bbcr-vkf8rz/operationStatuses/08584113986227757596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-42bbcr-2rk6t6mlf2lm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pgend26hbad42\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"34834e6a43449c92d460d1d939d02581ee8400228409138a2030fe12504b72a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"18a080cede1bb3983c6e834e6eebef5703de52cf674ed437a488782e09deeba7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-hlvkx99zwmlc/cluster-candidate-4-22-wjv59w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a580\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wjv59w: failed starting cluster creation \"cluster-candidate-4-22-wjv59w\" in resourcegroup=\"rg-candidate-4-22-hlvkx99zwmlc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hlvkx99zwmlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-wjv59w\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-hlvkx99zwmlc/cluster-candidate-4-22-wjv59w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a580\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wjv59w: failed starting cluster creation \"cluster-candidate-4-22-wjv59w\" in resourcegroup=\"rg-candidate-4-22-hlvkx99zwmlc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hlvkx99zwmlc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-wjv59w\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3a7d6bfd59f917701d2434c8f3edcc13afc349965e519f0db52dcdf74e7f452d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"df16ba4a6674c679f09b89c1eddcb51a1f9ca845693dcf206c2902823ef7e013","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94ac0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gtn8glz92n2v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gtn8glz92n2v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94ac0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gtn8glz92n2v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gtn8glz92n2v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3d450efd2fc3c2d06ffd8b06c246acc87ff97f898a62c92d53fdecd06eb3dad4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5ab72705e757a4285a503be51374c0f65a83934561314f0ab6a40725e9e8423f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-sjhtzvzlt9l4/cluster-candidate-4-23-5ld87l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02240\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-5ld87l: failed starting cluster creation \"cluster-candidate-4-23-5ld87l\" in resourcegroup=\"rg-candidate-4-23-sjhtzvzlt9l4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sjhtzvzlt9l4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-5ld87l\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-sjhtzvzlt9l4/cluster-candidate-4-23-5ld87l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02240\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-5ld87l: failed starting cluster creation \"cluster-candidate-4-23-5ld87l\" in resourcegroup=\"rg-candidate-4-23-sjhtzvzlt9l4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sjhtzvzlt9l4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-5ld87l\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3d9ae1869478522c0d660c0724f25433d3c8d6b5e17d50248b3abf0a94000685","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5743f1054d0ef1e157783ab637285ae718a794a73ec3f9d8e1cd66e8dc185998","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1e3a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-66dgzj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-66dgzj/operationStatuses/08584113987138933581\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kvx8wtp49bjz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1e3a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-66dgzj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-66dgzj/operationStatuses/08584113987138933581\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kvx8wtp49bjz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3e25d7b66970c4f9a0d74b81fc2e0e96b43e84c62ae71bfce888524ec17b5fd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"b3402bd39c1526399877b423af2058f9c4707b63ec0c5f879b3458d0c4b9860c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxmxrgw75sbn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fxmxrgw75sbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-fxmxrgw75sbn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fxmxrgw75sbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3e9800ed18863cad4e743efc93b39b54625aa3a6cf793b90ed1084b87a74aa7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"0b2e0e2a705eef2a8c34a9cbe9058b556f7e534770a547f7b0609260c02c4bfc","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c825a0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-xxnn2gk7lrb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-xxnn2gk7lrb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c825a0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-xxnn2gk7lrb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-xxnn2gk7lrb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c92605250abac180cc2107481fbee3c0820a6506cce092e3390df47fe7a4f27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"c2e9c1eb22fec11e3787804927eeed16352b79cdf1428605ce93e959bffaae09","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-xcgpz6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00c80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-xcgpz6-4zrvmh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-xcgpz6-4zrvmh/operationStatuses/08584113987705963996\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-xcgpz6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00c80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-xcgpz6-4zrvmh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-xcgpz6-4zrvmh/operationStatuses/08584113987705963996\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5132418cb688e0f464c20bf49263f11e3c56530cde9bef0a23a87ca17cd80cef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"d4bdde14f6cc55c0354fc8f6e82a55421657445ff0aba3cf3df52d2f44eda51b","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000712140\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-cl52xgnhc4wx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cl52xgnhc4wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000712140\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-cl52xgnhc4wx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cl52xgnhc4wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"537c3c549107faa55eca8cdac4edf5a79e8644962bd24f8b977c35629e70d846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"f3e5b048e00e5607edfcc1021209c6ad811e1b8fec6fd4f78740242a2b4ef145","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dm79nndfl6r5/cluster-candidate-5-0-x7bgf9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-x7bgf9: failed starting cluster creation \"cluster-candidate-5-0-x7bgf9\" in resourcegroup=\"rg-candidate-5-0-dm79nndfl6r5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-dm79nndfl6r5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-x7bgf9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-dm79nndfl6r5/cluster-candidate-5-0-x7bgf9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184e0e0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-x7bgf9: failed starting cluster creation \"cluster-candidate-5-0-x7bgf9\" in resourcegroup=\"rg-candidate-5-0-dm79nndfl6r5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-dm79nndfl6r5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-x7bgf9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"53f2d0a8ddc93d19fc03abbe55080ba87f336d35e6bb35cd1b5958c8d1bcc7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"578200e0d6235a987906427ba8a60a1f9bad1525471deb0bda8be5707a26c15a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-jrrftk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da6360\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-jrrftk-g7vqn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-jrrftk-g7vqn6/operationStatuses/08584113987434500608\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l442rxkb7vdp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mpj57e7od7fwq\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-jrrftk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da6360\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-jrrftk-g7vqn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-jrrftk-g7vqn6/operationStatuses/08584113987434500608\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l442rxkb7vdp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mpj57e7od7fwq\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"54b5b42e538f9bfd56a4b942ca64777757dbe2d67c6bb0b85b740d3e43cbb2bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"1658b955eb3643def75168d8be5b7cdd2bbbdffba30215221b837273eef86550","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-vvgtcx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cb00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-vvgtcx-tmrhxj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-vvgtcx-tmrhxj/operationStatuses/08584113987427336698\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-dpmbrwp9fml7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mem35qo2kwqbg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-vvgtcx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cb00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-vvgtcx-tmrhxj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-vvgtcx-tmrhxj/operationStatuses/08584113987427336698\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-dpmbrwp9fml7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mem35qo2kwqbg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"56c08c18a11bf22fb08616331c897e01e7965a46bf9f6c08836d1a1a41c7c401","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"d4b1caf980e450681931897e537f4227ce5d0899055d23d5f616842e4e73d6eb","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001120500\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-swjhtx-jx27v7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-swjhtx-jx27v7/operationStatuses/08584113986224996378\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-swjhtx-nll5kh9rqz25/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-sfpatc6wjwnlg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001120500\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-swjhtx-jx27v7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-swjhtx-jx27v7/operationStatuses/08584113986224996378\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-swjhtx-nll5kh9rqz25/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-sfpatc6wjwnlg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"603a09cb93686b166dee39f1dc0bfe3d422e3da352257a6e55aa4ee637160838","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"a3128fe89c2b34a47c935b73ccf2c1a9a2740878a1a3290746b393ecbb354616","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fec1e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-t4c48wqpwt72\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-t4c48wqpwt72/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fec1e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-t4c48wqpwt72\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-t4c48wqpwt72/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7462405d7855080423ca589a6bc2930c20205fc79d5c6f49a8e913e04c25996d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"0d143ad9d871f8761c894631d5ac5ac89f118ac56bd392b9fad3aef7ef29f7cd","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-v7kjhv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d905c0\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-v7kjhv\" in resourcegroup=\"admin-credential-lifecycle-test-qws7j9zzgwzk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-qws7j9zzgwzk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-v7kjhv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-v7kjhv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d905c0\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-v7kjhv\" in resourcegroup=\"admin-credential-lifecycle-test-qws7j9zzgwzk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-qws7j9zzgwzk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-v7kjhv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7664d378010bc912413c3238bd2af1ff0209a98794b06b1444bd5fd618d29860","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5e161d805837a94ed6359e3e20abe23d9621b72c256eeb4160963d2f2ded8efa","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8ef20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-f4gj82\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-f4gj82/operationStatuses/08584113987063020089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-j8rcf7j8sc84/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8ef20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-f4gj82\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-f4gj82/operationStatuses/08584113987063020089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-j8rcf7j8sc84/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7a111841552b4f85c2be6ebd96989a3817ff3720a4db5130552206dd1744534a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"e0d27c79c5899c123b80405656642920e76f42cb9309a89bffa6098039e41580","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101b980\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-6qq489-c5jsdg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-6qq489-c5jsdg/operationStatuses/08584113985901622419\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-6qq489-tpss6kb9nfnv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yslh3n7rfjni4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101b980\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-6qq489-c5jsdg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-6qq489-c5jsdg/operationStatuses/08584113985901622419\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-6qq489-tpss6kb9nfnv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yslh3n7rfjni4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7e5effd3fe1189ca7341cc5378266cd17d98d17906d289e9a3876d99df8d9ed1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"760124e7b25aec0cd4af5e7f97d46af297b8834a0ae47c0dcfff744bd3a49ba5","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9a2e0\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pptmdm5h47tr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-pptmdm5h47tr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9a2e0\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pptmdm5h47tr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-pptmdm5h47tr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"83c045168f59207475afff134bb25066c8e4aab253b543aedac3a1fa8e2da19c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"9b3fe187d08170fbc37adc61518c7eeb7b25a95802571b8983f126cabf6fdbc6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000e923f0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-z5x8hr8lv45s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000e923f0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-z5x8hr8lv45s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"840567d0426e702ecb571fb1e25517620ecdf7c1fc5739a9ba61845c6fed041e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"a243861fdf0dc90f346c36654a0f53ec596c9df69556eca343abdfd26653e306","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007876a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-lrqgv6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-lrqgv6/operationStatuses/08584113987055376578\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xh4nq445fjw9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007876a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-lrqgv6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-lrqgv6/operationStatuses/08584113987055376578\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xh4nq445fjw9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"84a85d09bb555094d7b1d5b0380987f12d54657c72824855fe28f60a24baec07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5ebbffe49b3cac6c858179fca1ea96cf96ef5a9cb1c80ecbbe6a096d6439ff4a","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00032e320\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-rnbmsq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-rnbmsq/operationStatuses/08584113987137499225\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-wcgtvsqfsxxx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00032e320\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-rnbmsq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-rnbmsq/operationStatuses/08584113987137499225\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-wcgtvsqfsxxx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"859f50e95656ea82c224188772b202a5a71d42fb2c6faf0eaabe887ca86f06bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"5853cc43cf7425055621ffb2ed66038b5546de23d7a240578f0080ea258e2b72","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00180e180\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-47vf99jdbhj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-47vf99jdbhj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00180e180\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-47vf99jdbhj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-47vf99jdbhj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"893f81fc3cba65c7e45c2e195ef96ba65ee698daeb3ac084bb6268f45acb4f21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"8ec3e712fd3e8543d2d9d2a3f6c13734045c633652dd3994ffae220c9f4487ca","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00051f700\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-mf589sk4rj9r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-mf589sk4rj9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00051f700\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-mf589sk4rj9r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-mf589sk4rj9r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8cf6c13d58dc38327acda503a20b7d6cfe82c02bf9794ae96cd2d83745cb0dc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"00d3d6c70857d58cc7a77ad3cbf60f9749e64131ae86e9c5bbed23a03414b07f","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001026280\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-shn7fn-zvxvfc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-shn7fn-zvxvfc/operationStatuses/08584113985869276090\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-shn7fn-psrg7w9f9xgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ev3yo2onirsd6\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001026280\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-shn7fn-zvxvfc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-shn7fn-zvxvfc/operationStatuses/08584113985869276090\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-shn7fn-psrg7w9f9xgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ev3yo2onirsd6\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8f23546f53d2263d810e536ab2e31c2d326bec1e6e73e5853570a838fc2c85a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"df4451d3e29bdc3a64d7d56a21b5e669cdcab64a57c00193734b831648f9892a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c18180\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-bwh9fhw68vj7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-bwh9fhw68vj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c18180\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-bwh9fhw68vj7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-bwh9fhw68vj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"92ad85ef02c88f9d1782bb56ef12490e244c27a170505a3bd5e7d47b4831edac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"76eeb452365a9197cccddb41b3c6454114a1f922f7775ab7586fdbf3211fc438","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e04180\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zwxh6x2kssck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-zwxh6x2kssck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e04180\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zwxh6x2kssck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-zwxh6x2kssck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"93441c897622c3e850f52758c518f445f7185df1c8a4660e2ef66acc7f1fb73d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"b06bc75230ebcfa423f62726ee3b474f1b5c932e5eb57e3f75869553b569f537","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d3560\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-d7mvztslkt8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-d7mvztslkt8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d3560\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-d7mvztslkt8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-d7mvztslkt8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"95d61bbc372497348960f785f76ca9d6a297939bf3bc8d8d5ac07a835973d774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"2789ba3cd7312b4b858ed6732db313a63af15c5443bb24ef97632db2fab0aa1a","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001092280\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f5hpwzxskrlb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-f5hpwzxskrlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001092280\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-f5hpwzxskrlb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-f5hpwzxskrlb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"99a086c9df4d50c20a4b039d3f1b6e09cb810460692cc0ee86a6f62962516caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"8a3707252faaf14736b2be9fdc31fbfb7ee6fd9a2c628081309edfdac8fa18a3","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1cc00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-w9k5pm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-w9k5pm/operationStatuses/08584113985685912165\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v8ln66whgcfk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1cc00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-w9k5pm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-w9k5pm/operationStatuses/08584113985685912165\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v8ln66whgcfk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a06379499b2a0b13cc75c0a45c560581a0751ce453cc2b782b4866dd01bd9191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"e403149dc0ab41b7958114f780c8dfec643d355ef165e66823690ee143d7dcbb","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000e81a40\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000e81a40\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a066cd2d42e65f9809e74a7ab94bae9a2016f9f6a41395d9d5981e25e3095744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"9f4ef32027d9d349f278002e9bcde48f6415631082b03813d234b189e64d7959","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098eaa0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9fk2x7xsncqp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-9fk2x7xsncqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098eaa0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9fk2x7xsncqp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-9fk2x7xsncqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a3458daba4af9b23da5ebd410d28cca0e01756947e3562bc60bc1456d2509c21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"a96addc801edde278066d55645bd6687bd80418d4c10c554605fa06785f260d4","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfaac0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-t75xkk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-t75xkk/operationStatuses/08584113985598746400\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-xrwtbrzrms4p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfaac0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-t75xkk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-t75xkk/operationStatuses/08584113985598746400\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-xrwtbrzrms4p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a90a355c3b894ac54f1ea927543bcf41e2ea010fb874baf0b73fdec645a4e938","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"103c847c3dc0e0c5a0c1e2041083d44a521ad4942917fb41297142b8bf729ae1","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000984920\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zmq2fmksvd6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zmq2fmksvd6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000984920\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-zmq2fmksvd6w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-zmq2fmksvd6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ac366795e8e0abbb6d2c5b1d47b08111f297f95f1d242ade417ec6f919a70927","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"72b7cdf642ce3d305cfdf2201f3999ecc6881553d5e2167443ac80f1a5e34e81","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ae00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-djz46v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-djz46v/operationStatuses/08584113987138680420\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-wpsbhgpdgqb8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ae00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-djz46v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-djz46v/operationStatuses/08584113987138680420\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-wpsbhgpdgqb8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bbca8d6c9649b18fa1f0603e2912ba5bbb0df6af18a4f710fec9a2fc8284ec89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"711a23fdc6554e3f11a83326688aeff48062de542e6bf7ed6bf7c7bb3c91aa4b","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba8c00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-5x6zpb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-5x6zpb/operationStatuses/08584113986762247712\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-cplcmv2sq2w7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba8c00\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-5x6zpb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-5x6zpb/operationStatuses/08584113986762247712\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-cplcmv2sq2w7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bdd85e235249070ee9f06884757ed2fd61e490812ff5c95ba2270d72fe1887cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"207076026b7204986c55fc7402443d46f35a2e59e1d8abc866ccf5d15030f36a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-nmd797\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001752920\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-nmd797-bjjz9t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-nmd797-bjjz9t/operationStatuses/08584113987415585751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-qtbdqjphjldj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xfkchgv3ef7bk\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-nmd797\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001752920\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-nmd797-bjjz9t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-nmd797-bjjz9t/operationStatuses/08584113987415585751\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-qtbdqjphjldj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xfkchgv3ef7bk\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c1b02cbc0ce600bda6ca00f4e7d67644aa0c826e98bff9b252edeeef7bf49d30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"a5caf3ca78309008f43557f4e7fa4ff03bf651fed81afe60cc0f43daae52ccfb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-45rstd with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000902360\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-45rstd: failed starting cluster creation \"np-version-upgrade-cluster-45rstd\" in resourcegroup=\"rg-np-version-upgrade-45rstd-nh6m8src8lfr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-45rstd-nh6m8src8lfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-45rstd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-45rstd with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000902360\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-45rstd: failed starting cluster creation \"np-version-upgrade-cluster-45rstd\" in resourcegroup=\"rg-np-version-upgrade-45rstd-nh6m8src8lfr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-45rstd-nh6m8src8lfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-45rstd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c4458708a76bcc4b6a96a7996179bcaa51c3a56ce02802519874984ccea98c46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"3c6d3bbf620206422ebf37e0988112e383deeab2de78cc2c3d71820ae7402c1f","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012820e0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nbwkpxzhms7s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nbwkpxzhms7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012820e0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-nbwkpxzhms7s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nbwkpxzhms7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c6a0f4ab2da61e219b329919878f70b539dedadef6acb15bdfd5af15e655e83d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"2d2fb06c8452a100942b711accf2add79c08e2ad05176213f3e8e83c23c10eb1","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-ntq4zw\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ce260\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-ntq4zw: failed starting cluster creation \"cluster-zstream-5-0-ntq4zw\" in resourcegroup=\"rg-zstream-upgrade-5-0-hqk75qtw9hlt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-hqk75qtw9hlt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-ntq4zw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-ntq4zw\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011ce260\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-ntq4zw: failed starting cluster creation \"cluster-zstream-5-0-ntq4zw\" in resourcegroup=\"rg-zstream-upgrade-5-0-hqk75qtw9hlt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-hqk75qtw9hlt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-ntq4zw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c9263f2bf5989b780c2e0a3efe5f6bbe374741a4e0bf05690d1df3a0b3118072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"2577d97b4cfb09d0e29fa89cfbebf02bbe5521e4fedc6b8305c75d3a21b78bd1","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b04c80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-2jwn4b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-2jwn4b/operationStatuses/08584113985700308617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-rczl9wf28lwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b04c80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-2jwn4b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-2jwn4b/operationStatuses/08584113985700308617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-rczl9wf28lwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cb50b37d0f9364f4a2211faa4e506579f5220d76e6e7b93cf9b64231227a7579","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"b7d497a59299e6705ec5c27d0954476df3b73d478d675970b241ac6bdeb0beb4","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d10500\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-l44lntsvmpb8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-l44lntsvmpb8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d10500\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-l44lntsvmpb8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-l44lntsvmpb8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cc9aa85ebcc9e7bade7ade3aaefbfaae020ee5694fa0cc0378424aaf62d3a7d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"62b8ed0f3c7495d4a962b220108f6cdf7916768f686fc9f71620743b5266548f","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111e320\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-4bqqhnwdh8r5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-4bqqhnwdh8r5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111e320\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-4bqqhnwdh8r5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-4bqqhnwdh8r5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e5dddba8d658c08005228998c98bdcb0862330a784d0a064348028bc4d947c48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"d2c063f2267dd2c0079f8d444aad43b12dcd51b7cb090dcd5fcfa03d5d526369","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408980\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-z8pbtk-44k8bq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-z8pbtk-44k8bq/operationStatuses/08584113985924298427\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-z8pbtk-p9lrjqz5xlmf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-a44nzwnsonewu\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408980\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-z8pbtk-44k8bq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-z8pbtk-44k8bq/operationStatuses/08584113985924298427\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-z8pbtk-p9lrjqz5xlmf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-a44nzwnsonewu\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ee5e01fe6f3028453a5720afefca1743417e250c168ebc174de4059fda8ad08d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"04f03534a2c11433842c187808432c4fe56b5e127cb74b63d712bb47709a5195","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001180e80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-rdm6f7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-rdm6f7/operationStatuses/08584113987041544035\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-pxh9tsgnpxwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001180e80\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-rdm6f7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-rdm6f7/operationStatuses/08584113987041544035\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-pxh9tsgnpxwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f3c1dcb1ec0d2293501fb19508fd6ef1fb657fc4cbf8a44aea53c56aeec3c452","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"66bb238f691dab83756a60fd4ef63b42d1bcad07037abbae7bfe4492035f5ace","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-6sv9qtzkp7p5/cluster-candidate-4-20-k9rjvx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d46cc0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k9rjvx: failed starting cluster creation \"cluster-candidate-4-20-k9rjvx\" in resourcegroup=\"rg-candidate-4-20-6sv9qtzkp7p5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6sv9qtzkp7p5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-k9rjvx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-6sv9qtzkp7p5/cluster-candidate-4-20-k9rjvx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d46cc0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k9rjvx: failed starting cluster creation \"cluster-candidate-4-20-k9rjvx\" in resourcegroup=\"rg-candidate-4-20-6sv9qtzkp7p5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6sv9qtzkp7p5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-k9rjvx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f4faa05cc9402b463df056c3bbae64767f8b7d79c44da7468ab823e4e642c1e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"29b4f2a9dcd277b5f12403d713f2be8b0efe8f05e42b48c6fc64c2d68594b63d","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006d0900\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-z4tc6h-tk4t87\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-z4tc6h-tk4t87/operationStatuses/08584113985739549250\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-z4tc6h-rxbv2s7dz2bg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-h2mxcxvipvybg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006d0900\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-z4tc6h-tk4t87\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-z4tc6h-tk4t87/operationStatuses/08584113985739549250\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-z4tc6h-rxbv2s7dz2bg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-h2mxcxvipvybg\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f9ee2519bf4a8c1006fcb1d15ab309f1761721fce26aec3e22412815c32abbc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102904467069341696","occurred_at":"2026-09-23T23:34:29Z","failed_at":"e2e","signature_id":"b5d93b8bde9091dbdbd0277f382eef3ce49ab826cf8e62f6b226b70f017f85a5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-t78srk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015fbe20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-t78srk-ftnpvj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-t78srk-ftnpvj/operationStatuses/08584113987507742946\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-v9krnbjlzvgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-gdp5fqxbc7uf4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-t78srk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015fbe20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-t78srk-ftnpvj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-t78srk-ftnpvj/operationStatuses/08584113987507742946\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-v9krnbjlzvgl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-gdp5fqxbc7uf4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":58,"failure_pattern_ids":["20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":58,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T23:00:13Z","started_at":"2026-09-23T23:00:13Z","completed_at":"2026-09-23T23:15:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:26:42.941802063Z","timing_metadata_checked_at":"2026-09-23T23:26:42.941802063Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:26:42.945905845Z","region_metadata_checked_at":"2026-09-23T23:26:42.945905845Z"},"failed_at":["e2e"],"failed_test_count":52,"occurrences":[{"row_id":"0292f2a116f6c1ddff4750e3150ac9f18879cd68777f9064f95c4e8185906b9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"ed35adaa8e06ba709899bb9363d50812ad914c59c4485a7dedf2f384e127f581","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039efc0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t9ljlvhpqrl2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t9ljlvhpqrl2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039efc0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-t9ljlvhpqrl2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-t9ljlvhpqrl2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0575b8cf6d1ce3c7d28dbd4b635b0f796d329edce8dea5ceacb9f308864499ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"e0c1ab0a923b3315a04331d9fd43f31fc3b87e0ba01d6d856f153ac9ab08804f","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wk479hrrr6dd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wk479hrrr6dd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wk479hrrr6dd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wk479hrrr6dd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"09027cd6e6ffeb4dd617361ca96b1f32eebee3baae052bf4a1fef9938c70f259","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"7921192492632722db01f265f3803c9c63f3df67888105c25cde9c26f19b4bb4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0c280\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-n2mwb88rc2kc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-n2mwb88rc2kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0c280\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-n2mwb88rc2kc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-n2mwb88rc2kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"09ba7629d3463880a3668bd0769b4980f23821fdad724e9455594bffcb1fd094","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"1e11434627c662dbbe902599e9237bb9f801d7b594c61e3714ab1bc17eca9161","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5960\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-z4z66hnzq25r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-z4z66hnzq25r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5960\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-z4z66hnzq25r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-z4z66hnzq25r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0b3a59ae86eb11d03db5f4e1be23bb524f8c9421b6ee69605e06f4a9cd42d075","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"e6c51f65ab36666b3dcc2d0da28446428775135469fa57e5d425afa8e424a2a5","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015bb5f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015bb5f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0b4c9cecc5a7f4a2aad4397220604340b83570070bbb8d8c663c3cb6017ef364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"c623dc307c56fc7aabf82a0862f91773eb11d3a0736caf6f633dfbc71a33cbd4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-gvq7bc\" at install version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e580\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-gvq7bc: failed starting cluster creation \"cp-ystream-upgrade-5-0-gvq7bc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-5gq89l7l9zj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-5gq89l7l9zj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-gvq7bc\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-gvq7bc\" at install version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e580\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-gvq7bc: failed starting cluster creation \"cp-ystream-upgrade-5-0-gvq7bc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-5gq89l7l9zj4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-5gq89l7l9zj4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-gvq7bc\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"11326b5dd53b6af133f59f1d6abd9ed1cc55d90c48ba64f71a119913d43b79d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"fd4fd6cc3f3b5dcc77f5c5745a3117eaf3d30f9f2b15b40a36c3cae3ffdee395","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b582e0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-blllqrwdhwg2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-blllqrwdhwg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b582e0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-blllqrwdhwg2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-blllqrwdhwg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"118bc0df4b39a9f23cf87b3529244c52c8cc1bfc9245947c0ace472d4f1a9b00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"bd7a905f012148451aac90fdee9e8bce962b34e00b90af1ab067a26e5e89f381","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jrntrv2b4mzw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jrntrv2b4mzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f10140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jrntrv2b4mzw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jrntrv2b4mzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"16d9df2f0c9cc1c45e68948ee8b2287028c489bab5e3a4d6883010961f2b6150","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"9c9ed5a78aed1b362d5aeec8761fb5cebdedeafacf061840b8e80db9cf7b0faf","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12180\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-j75pdr9hg97t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-j75pdr9hg97t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12180\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-j75pdr9hg97t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-j75pdr9hg97t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1b7affbc7c0cb17e0ff397a36839db2ed9b55b2dc60ab3a7989d98cbd6688529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"6734cdcb4d8aef16e200ba287bf768f9fdd46715bf7a03214569d57b945db659","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001072140\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5v7nmx4glfms\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5v7nmx4glfms/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001072140\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5v7nmx4glfms\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5v7nmx4glfms/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"217d7efce024cd5d62319d7647422744e1af4ce44e73272c0fb96970d19b4573","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"4128486c577bef73cb4bbdeb9e75cb7a1aa87e84c86bf43283309add2a1f3286","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14640\u003e: \n    failed to create HCP cluster ea-list9flw4f: failed starting cluster creation \"ea-list9flw4f\" in resourcegroup=\"ea-list-89wl2cn26v96\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-89wl2cn26v96/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list9flw4f\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14640\u003e: \n    failed to create HCP cluster ea-list9flw4f: failed starting cluster creation \"ea-list9flw4f\" in resourcegroup=\"ea-list-89wl2cn26v96\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-89wl2cn26v96/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list9flw4f\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"238f7084d08646ac41a9cd30492faf768c5d23320099976f3cb1dd792e3055ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"50239d603bb780e2e003b8ea21c9046e3218d0cfe34261f655d2951ccb605eb8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-rckglw\" at install version 4.21.0-0.nightly-multi-2026-09-23-170748\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069a7a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-rckglw: failed starting cluster creation \"cp-ystream-upgrade-4-22-rckglw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7mwhqknsl2vx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7mwhqknsl2vx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-rckglw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-rckglw\" at install version 4.21.0-0.nightly-multi-2026-09-23-170748\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069a7a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-rckglw: failed starting cluster creation \"cp-ystream-upgrade-4-22-rckglw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7mwhqknsl2vx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7mwhqknsl2vx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-rckglw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2b952625c0adf5f26c34eb9118de2618e4bcf4146ea9371580ca730075aa4650","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"22d721cd87f00c16c308d65485d8483188769f0546a555d083b414ba8263727e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f166c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fzxrb9z9hsxg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fzxrb9z9hsxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f166c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-fzxrb9z9hsxg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fzxrb9z9hsxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2d7119165abbccbc49ed96220474c402bd923de0c22cb807263ce3a8b990081c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"15fc45cd3a6e9b691b043a24977de024edb4d4590b2e83405ee977a707b3bfbd","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-92b6l7\" at install version 5.0.0-0.nightly-multi-2026-09-23-031212\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e642c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-92b6l7: failed starting cluster creation \"cp-ystream-upgrade-5-1-92b6l7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-8pm7wghtpk6v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-8pm7wghtpk6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-92b6l7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-92b6l7\" at install version 5.0.0-0.nightly-multi-2026-09-23-031212\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e642c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-92b6l7: failed starting cluster creation \"cp-ystream-upgrade-5-1-92b6l7\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-8pm7wghtpk6v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-8pm7wghtpk6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-92b6l7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"31950710a1ba435bbbc28a567b86e81b0f84e132f270c04cd786501d459f225a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"1c2d92e2bedb33fe555fec70451124e32fdf767ba423de0c8f828af1bc35cc8e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-fqch49 at version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9c380\u003e: \n    failed to create HCP cluster np-skip-minor-fqch49: failed starting cluster creation \"np-skip-minor-fqch49\" in resourcegroup=\"rg-np-skip-minor-fqch49-rrvslkw6xp2h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-fqch49-rrvslkw6xp2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-fqch49\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-fqch49 at version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9c380\u003e: \n    failed to create HCP cluster np-skip-minor-fqch49: failed starting cluster creation \"np-skip-minor-fqch49\" in resourcegroup=\"rg-np-skip-minor-fqch49-rrvslkw6xp2h\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-fqch49-rrvslkw6xp2h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-fqch49\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3f0cf2867c64d24b5ce7b7bdfd4b6671805226273aa09eb92557d3a3e78470f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"a597ae1c8f906eafb58886f6816c5618cf09b5cf8ed7636f9cb0360f165e8bed","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b5a2a0\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-gfwfk6s8b9j2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-gfwfk6s8b9j2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b5a2a0\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-gfwfk6s8b9j2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-gfwfk6s8b9j2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"48d23afac9b63c1edce951cf7cba7250081d7393f968736d280828c3d899b266","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"460443b8e000eb1965f80fb4652248f0dd60ecdb22f5297b96d6782ac13c0d1a","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000912300\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nmfgpbvxrwtl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-nmfgpbvxrwtl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000912300\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nmfgpbvxrwtl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-nmfgpbvxrwtl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4cb3df7a7c0b42c2d3aaa182660d501316044bb9ab428c43db3a4d993ba69871","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"b64a093bb316ed23148a238480d3c3b297377eec6a0d5b6477f4d2ab1d28f450","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d8200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-fs6xcx4wqfx6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-fs6xcx4wqfx6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d8200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-fs6xcx4wqfx6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-fs6xcx4wqfx6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"55460b6913846de612274d4ff165bc4fc32140c5b8121fcf87cd2cf3cc256499","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"2d025d742d960f65ce0df017beb150ac204ad79db74000677aa6ae4a6d667007","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f16180\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-p4696z94nm5m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-p4696z94nm5m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f16180\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-p4696z94nm5m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-p4696z94nm5m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5fd9167ff4a712c976f57f327d8b20c18f61b5502245292d74f3f128caa832cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"837390542bdc443991e02ac2fc910c99edc2dbc38f20395d7af528a232fc43e9","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001522260\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-97cn59jkhhl4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-97cn59jkhhl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001522260\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-97cn59jkhhl4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-97cn59jkhhl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6925289723acdd38df122e7e5f1069e104fe9c18a46e58dbd1dae925749ae6db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"29c1e995b8db1156d89e071660ed02b25db28529121ba2f957571744517c5a69","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7280\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-v6cldcx5jl9f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-v6cldcx5jl9f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7280\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-v6cldcx5jl9f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-v6cldcx5jl9f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6bd31aa2443c575cdd39feb66006c67621035190eccff318f5a18367f89b3bd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"6f1244f44ecd8b4386870618b25b694425b1ab47b331ebc35c4fe4bd67d4a631","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe2120\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tq7lzxq6ld4m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-tq7lzxq6ld4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe2120\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tq7lzxq6ld4m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-tq7lzxq6ld4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6be5b8e6721e335df5ea3e63636bff326bd956f566b7ba9bd19a0a89eb7a2d98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"d4353e2de7ecccab93779b4bcfa87a57824ecdd3130b162614c71c910af28701","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9c2a0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-xq29bq2p4r67\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-xq29bq2p4r67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9c2a0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-xq29bq2p4r67\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-xq29bq2p4r67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6c15fa8469336c829f320d85eaa0d758c40a6921526cf4074ab3d89bc047d571","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"a068ba3d18cd23c9aba119652965396d00f35a6fa23cfaacf2c15e2793881de7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-t2bxq8nv5246/cluster-nightly-4-23-p9fb96 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b28880\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-p9fb96: failed starting cluster creation \"cluster-nightly-4-23-p9fb96\" in resourcegroup=\"rg-nightly-4-23-t2bxq8nv5246\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-t2bxq8nv5246/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-p9fb96\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-23-t2bxq8nv5246/cluster-nightly-4-23-p9fb96 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b28880\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-p9fb96: failed starting cluster creation \"cluster-nightly-4-23-p9fb96\" in resourcegroup=\"rg-nightly-4-23-t2bxq8nv5246\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-t2bxq8nv5246/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-p9fb96\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6d5c237352af6e6801c95d2b964e045cfae3631af4016b724c3ac0405b6005f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"378fe00b23b03f756b8692d9b32ec7d2be8def98bef3a4def31f682ef75a69e8","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008843e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-74cvpvwnrmg8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-74cvpvwnrmg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008843e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-74cvpvwnrmg8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-74cvpvwnrmg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7574272e76bcfad4f86ca38bd268d0aeb8a33eba77ba3efda8824cf1651cb873","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"9f7fbf5b4c12c9689dde71f7903041c031a8c2c9d7c2349fffe2cf45bc4705b5","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d265e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jwdzg46qwq7w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jwdzg46qwq7w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d265e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-jwdzg46qwq7w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jwdzg46qwq7w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7da2c71080f5c8af60aa447b8e18d7b292eda7dcb2deeadcba7491546de26bca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"e707ce73c3444549cb1b45692d1a7984bcddc94e2b321c8ef6f32f988e83b46a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000457f20\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dbdgjlmvwgtx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dbdgjlmvwgtx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000457f20\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dbdgjlmvwgtx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-dbdgjlmvwgtx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7e9d0e8b206c16f28eee6f86b42164d33e616e00da6b85ef9f8adba40d931cdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"67ea86669999abbb4487fef31f427939e9e272ebfccd1ab225343a4011bef471","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-npwjgprz6msl/cluster-nightly-5-0-hfdrrw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00108e140\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hfdrrw: failed starting cluster creation \"cluster-nightly-5-0-hfdrrw\" in resourcegroup=\"rg-nightly-5-0-npwjgprz6msl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-npwjgprz6msl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-hfdrrw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-0-npwjgprz6msl/cluster-nightly-5-0-hfdrrw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00108e140\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hfdrrw: failed starting cluster creation \"cluster-nightly-5-0-hfdrrw\" in resourcegroup=\"rg-nightly-5-0-npwjgprz6msl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-npwjgprz6msl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-hfdrrw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7fb6299b9ab07d29c147297e816654cb1095a714e1df2a7e376b5daae27ede59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"85546bd52d8389ea58add506b472388cf04ba0e5467d3dc4d5f9a09d41818886","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000736180\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-kszfsz4kjdp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kszfsz4kjdp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000736180\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-kszfsz4kjdp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kszfsz4kjdp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"903901825e2b609e6e4c2164bb2c43db6df8fb14a354e07526598d65b5d5a74e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"88fdaade50cf0d1f17b365e4c98e2ce7167b37da529946fb1b6ce451043f8534","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-khv8dvr7ks7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-khv8dvr7ks7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-khv8dvr7ks7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-khv8dvr7ks7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"92f52eb873067dff708bbc9f91dfba8b07e1a23672c3decb1c6282145a3b5117","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"ee1260742cfd6d8b1b7904c0717f61165015ad94e818f219111800249f781b4e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-9xw8wplswg6p/cluster-nightly-4-21-4bm8nq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a941c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-4bm8nq: failed starting cluster creation \"cluster-nightly-4-21-4bm8nq\" in resourcegroup=\"rg-nightly-4-21-9xw8wplswg6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-9xw8wplswg6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-4bm8nq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-9xw8wplswg6p/cluster-nightly-4-21-4bm8nq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a941c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-4bm8nq: failed starting cluster creation \"cluster-nightly-4-21-4bm8nq\" in resourcegroup=\"rg-nightly-4-21-9xw8wplswg6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-9xw8wplswg6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-4bm8nq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"952f0c37f4ca5cd0fd1e93ddd11b55130f75fa84fb048ef59f82ca06e02c2297","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"422f4827576f5663abf88da2acea909e13a8a93903be9e159726618aafddf3df","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-c65rwd\" at install version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e162c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-c65rwd: failed starting cluster creation \"cp-ystream-upgrade-4-23-c65rwd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xc5ljtzh9ws2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-xc5ljtzh9ws2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-c65rwd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-c65rwd\" at install version 4.22.0-0.nightly-multi-2026-09-23-151735\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e162c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-c65rwd: failed starting cluster creation \"cp-ystream-upgrade-4-23-c65rwd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xc5ljtzh9ws2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-xc5ljtzh9ws2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-c65rwd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"96982163e4e822ec5e2a587dc413ff8e9943b75f8d6d190fb2e6e77b5bcbaf02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"177d3a6c2c33493e767ecaca9b49e225e0b353fc1f73f3a7f19b8d1d42de2b31","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdm9fjw2dttb/cluster-nightly-5-1-v74km7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-v74km7: failed starting cluster creation \"cluster-nightly-5-1-v74km7\" in resourcegroup=\"rg-nightly-5-1-vdm9fjw2dttb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-vdm9fjw2dttb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-v74km7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdm9fjw2dttb/cluster-nightly-5-1-v74km7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-v74km7: failed starting cluster creation \"cluster-nightly-5-1-v74km7\" in resourcegroup=\"rg-nightly-5-1-vdm9fjw2dttb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-vdm9fjw2dttb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-v74km7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"98359883e8fa53f9e3086a075d2186d4087a454e2a9dc69b001e57b38b53c37c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"2216802953356f2ab624815ca80f12ca32ed7a86476d153301b4ff00b4a54b22","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b182c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-p2sz9v5k8jtd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-p2sz9v5k8jtd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b182c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-p2sz9v5k8jtd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-p2sz9v5k8jtd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9b9a2a4ccb192b2e14affbe8600f5920863e7726696a76312e2a14e4eb111073","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"86320aedbaa0e91f0fa200e2296801441f3c0f977b35ca4e9212e9d6e4951e15","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-bgpnp8wsw6xn/cluster-nightly-4-22-ld22vq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f241c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-ld22vq: failed starting cluster creation \"cluster-nightly-4-22-ld22vq\" in resourcegroup=\"rg-nightly-4-22-bgpnp8wsw6xn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-bgpnp8wsw6xn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-ld22vq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-bgpnp8wsw6xn/cluster-nightly-4-22-ld22vq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f241c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-ld22vq: failed starting cluster creation \"cluster-nightly-4-22-ld22vq\" in resourcegroup=\"rg-nightly-4-22-bgpnp8wsw6xn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-bgpnp8wsw6xn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-ld22vq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a76176b1830a7566db8986b7d189f6d30cfebbe91624714b3f17ee9caf8b4a52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"3fc7356f7fe16e5e609d0ba5038de5a411cf304f107f57ef9c728c54c87f3726","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4zq985\" at install version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150e0e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-4zq985: failed starting cluster creation \"cp-ystream-upgrade-4-21-4zq985\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2t5stzd262x4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-2t5stzd262x4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-4zq985\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4zq985\" at install version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150e0e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-4zq985: failed starting cluster creation \"cp-ystream-upgrade-4-21-4zq985\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-2t5stzd262x4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-2t5stzd262x4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-4zq985\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a81885b15a48aa1ab2ed79a4f54f431d57396c6d428948f1ff0e4591d148d7d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"ef11ab2cf959392657372fa0909fc2c548315a7d685b35f76171056b4abcc194","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c8360\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-g7klgrs9fgjl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-g7klgrs9fgjl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c8360\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-g7klgrs9fgjl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-g7klgrs9fgjl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a97ab7e67995e781d86b269ae1bd070ef9b1c2e5377dffda92c393e7e885ca73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"0cccb5c6694b8a6067983c3b7bf775d7a351d03d11aaae0408425a37126b77b6","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001360160\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zht9gtvbf76b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zht9gtvbf76b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001360160\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zht9gtvbf76b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zht9gtvbf76b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aafca677319f5cd7ce4e3379e41a2b6862fca1dc9ce6d18b081970a7c7f0c5b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"be9719f4635bc456af0045a5b461c223ac5096f83cf7ae4dfc406fd8d8ea51ac","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000980380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-xw7mgxvr7szh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xw7mgxvr7szh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000980380\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-xw7mgxvr7szh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-xw7mgxvr7szh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b0bc05e18698b49e204718b4a7b071452de3e47dbc732b8d6d33b349d6650a08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"c70dc9b2344a22c84855927d709917851252038a14a65b1b42ef8df388ce4408","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0023de0e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-295h87tpx2wb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-295h87tpx2wb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0023de0e0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-295h87tpx2wb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-295h87tpx2wb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bf0a4d8e349c9457d63c1ba65d46d29b35fc35561791f411b1d0d389b4482f43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"8ebb44433de75ae1a59b237d32db8b6690cca6192875ac16af87e54b2f413ceb","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e5d140\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ljcwvrrqzgb6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-ljcwvrrqzgb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e5d140\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ljcwvrrqzgb6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-ljcwvrrqzgb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cc04956540b9e43fc7b7d83629c6ad8ae49b29ded81b5717862e2458b6f4f759","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"ab4cbe036556b338289f7725734bbb513d944dccdc26d6efeb46fec80dfa5a8c","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f64220\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-nj4mbs7m2xnq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-nj4mbs7m2xnq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f64220\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-nj4mbs7m2xnq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-nj4mbs7m2xnq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ccad193d160ab0e384038e4db9dbeab266b9174d81957294e42f4ea01c06f4c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"47aecd3202086479a3f88dfa9da71e860867c259c29f1b085fdff23422a0e308","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a240\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzwdzblv4lfz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-tzwdzblv4lfz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101a240\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-tzwdzblv4lfz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-tzwdzblv4lfz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d08fcddf8a7599847c41e506965b6c20034cc8e3c37ee10989e91893d2bc5ea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"1f20143686241c78e93d2d29986f7163132c10ed802ef6e532c64a395fa4d866","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab6240\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-s72gl49bdlhm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-s72gl49bdlhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab6240\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-s72gl49bdlhm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-s72gl49bdlhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d5c0aa3a777f5d894ccac56bc82280037570373b0adb0def50337082640dd891","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"93293e6b4fe69ff746741e539bee57e0ae0499574cb09083e0dab652c3f3b297","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ac320\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d2dfnkr542l8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-d2dfnkr542l8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ac320\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d2dfnkr542l8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-d2dfnkr542l8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"daaacb00a361bcb089770c940bf941a108897e0d5a9bf41dea2dfb4c9f03b41b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"310fff41d54de3202e5f9f4c3155139ebbac3cbc401622dae98eb10133d7a4af","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c46330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-b62sjvpqvlrq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c46330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-b62sjvpqvlrq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ded8a85f162dcabebfbddc776ef7d59b229b4d430ab3cc1f8c952b165043f0a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"055fc0966640e70d474c53a699b885b63dd8c74064a9e17fb602a8300375a12f","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00016cbc0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-57hgcc4bw6nt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-57hgcc4bw6nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00016cbc0\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-57hgcc4bw6nt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-57hgcc4bw6nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e3bd13102aaabecf003211f67f1148fb7e8013bfb890d867aa8921ebeff73034","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"31b6db9373de5fd907ddd06fb828d6f181f6a35afd6c454485818fc463598c5e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skk87w with version 4.21.0-0.nightly-multi-2026-09-23-170748\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e62b20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skk87w: failed starting cluster creation \"np-version-upgrade-cluster-skk87w\" in resourcegroup=\"rg-np-version-upgrade-skk87w-cq5mh9x4f6m8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-skk87w-cq5mh9x4f6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-skk87w\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skk87w with version 4.21.0-0.nightly-multi-2026-09-23-170748\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e62b20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skk87w: failed starting cluster creation \"np-version-upgrade-cluster-skk87w\" in resourcegroup=\"rg-np-version-upgrade-skk87w-cq5mh9x4f6m8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-skk87w-cq5mh9x4f6m8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-skk87w\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ef2a93bb1ef258bc18df761648aa3d0714997cd147f5b193905f75c4ae865bdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"1dd2c6cc8e4f394a34e0fc7e8213f2ab03bd15a43e139005281c4de11c90ccc3","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfc0e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6klmwqw9sdwc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-6klmwqw9sdwc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfc0e0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6klmwqw9sdwc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-6klmwqw9sdwc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f00207d8357f603d6e7a6d92f6140981d414983034a18b8287d98b4ade58dd1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"d9d382c4357a2f83dd034a43e9ce78fdc4093d1bcef932b3734fb4e2aa054e47","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-jjzc8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000761f00\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-jjzc8b\" in resourcegroup=\"admin-credential-lifecycle-test-hb7c2sfbh46t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hb7c2sfbh46t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-jjzc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-jjzc8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000761f00\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-jjzc8b\" in resourcegroup=\"admin-credential-lifecycle-test-hb7c2sfbh46t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-hb7c2sfbh46t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-jjzc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f1cab8d96d827257e63a53754db0442517f1a09dedddb78672cff6718e82cba8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"41eb5f994f3ddf34344ce72d61df3f563d8898b4a6450b02169b782482eea506","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df0260\u003e: \n    failed to create HCP cluster np-dg-minor-jcknfn: failed starting cluster creation \"np-dg-minor-jcknfn\" in resourcegroup=\"rg-np-dg-minor-jcknfn-jp7kllql2h9n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-jcknfn-jp7kllql2h9n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-jcknfn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df0260\u003e: \n    failed to create HCP cluster np-dg-minor-jcknfn: failed starting cluster creation \"np-dg-minor-jcknfn\" in resourcegroup=\"rg-np-dg-minor-jcknfn-jp7kllql2h9n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-jcknfn-jp7kllql2h9n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-jcknfn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f4d6eaed0425d726ee79a1665293c366d181704911a53418afb997471af5df13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2102895842175225856","occurred_at":"2026-09-23T23:00:13Z","failed_at":"e2e","signature_id":"d9b4da1e3652b9dbb42131926eb40a3f68dc2918cb70eb3c61df45ff7270e18f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-xflblvj4b79l/cluster-nightly-4-20-hp5qw6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8a340\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-hp5qw6: failed starting cluster creation \"cluster-nightly-4-20-hp5qw6\" in resourcegroup=\"rg-nightly-4-20-xflblvj4b79l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xflblvj4b79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-hp5qw6\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-20-xflblvj4b79l/cluster-nightly-4-20-hp5qw6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8a340\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-hp5qw6: failed starting cluster creation \"cluster-nightly-4-20-hp5qw6\" in resourcegroup=\"rg-nightly-4-20-xflblvj4b79l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xflblvj4b79l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-hp5qw6\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T17:20:30Z","started_at":"2026-09-23T17:20:30Z","completed_at":"2026-09-23T17:59:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:10:55.953128578Z","timing_metadata_checked_at":"2026-09-23T18:10:55.953128578Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:10:55.956176968Z","region_metadata_checked_at":"2026-09-23T18:10:55.956176968Z"},"failed_at":["e2e"],"failed_test_count":58,"occurrences":[{"row_id":"0955be7bcbd09eb54cff33aabb1e15fdeb0e13c9c087b5c8420da12a12e75bdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"cc7a5cd5ee8206e6d20ce4a924f1368feeab1325081fef357d5c935b3b9d6af4","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea63a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-md9bxlt4h2bl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-md9bxlt4h2bl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea63a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-md9bxlt4h2bl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-md9bxlt4h2bl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0994f3005d517820f823aa7813e48255e0886703aa39828a7359c354b943a8c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"2d9c7e2f49c360d0ece72f03034d96b551f5b6fc5126e16052ad10f272e652b2","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac62a0\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-kr4qllwsq4zw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kr4qllwsq4zw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac62a0\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-kr4qllwsq4zw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-kr4qllwsq4zw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0dcd4e6f4f88b0eb860b91a52415e8f7fed69da2d434f25e2e57e1f955a06ed3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"acba9062b228abb646f0453b339381dd60f3095dcd2638eec0023c8404f155f5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008a26c0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gwq5rnfvqbmj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gwq5rnfvqbmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008a26c0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gwq5rnfvqbmj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gwq5rnfvqbmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"13926f10babcb569667218c8b8803f19a48fad0f97a95b4c0e805ae7a86a76f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"f986660f71fd5e2eff4c2a2d31f4185e824ad6773c3a62840cd61cdbf74954e9","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-f9mwlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00014aa00\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-f9mwlb\" in resourcegroup=\"admin-credential-lifecycle-test-jnk69g62fsqq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-jnk69g62fsqq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-f9mwlb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-f9mwlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00014aa00\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-f9mwlb\" in resourcegroup=\"admin-credential-lifecycle-test-jnk69g62fsqq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-jnk69g62fsqq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-f9mwlb\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1511432a0c02e75a54d93d0447e72648859984bd225e02aa79d6b5d5c80d0a9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"c207cf1d189d0d3cf54778aacedacfba7248221d8883dbb57f709247ee506b89","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-rdjnfw with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009944c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-rdjnfw: failed starting cluster creation \"np-version-upgrade-cluster-rdjnfw\" in resourcegroup=\"rg-np-version-upgrade-rdjnfw-5m2htlhcktvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rdjnfw-5m2htlhcktvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rdjnfw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-rdjnfw with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009944c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-rdjnfw: failed starting cluster creation \"np-version-upgrade-cluster-rdjnfw\" in resourcegroup=\"rg-np-version-upgrade-rdjnfw-5m2htlhcktvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rdjnfw-5m2htlhcktvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rdjnfw\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1a699b9e7017e6cf15edffaf4da2bcedb6a479ebc5c24fdc9e2cda39063cb7ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"30a0cd932f16b4934c6c04f40a2551a1a8dfd6f1c7e9b6c8a18ef85340a6c9fa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-mr648lrwtr8j/cluster-candidate-5-0-jdjc8h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001926140\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-jdjc8h: failed starting cluster creation \"cluster-candidate-5-0-jdjc8h\" in resourcegroup=\"rg-candidate-5-0-mr648lrwtr8j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mr648lrwtr8j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-jdjc8h\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-mr648lrwtr8j/cluster-candidate-5-0-jdjc8h should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001926140\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-jdjc8h: failed starting cluster creation \"cluster-candidate-5-0-jdjc8h\" in resourcegroup=\"rg-candidate-5-0-mr648lrwtr8j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mr648lrwtr8j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-jdjc8h\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1e0d340b238c0a9540014fecc03fc132ef4c362e10175a3c3a5f7dc249e61182","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"2e7c66dd5b354188df699a79024b9c9df5f0658bb90ca1867f9faee1ea20cf9f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7a320\u003e: \n    failed to create HCP cluster ea-listdnmz6c: failed starting cluster creation \"ea-listdnmz6c\" in resourcegroup=\"ea-list-2tprnprxzp8c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-2tprnprxzp8c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listdnmz6c\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7a320\u003e: \n    failed to create HCP cluster ea-listdnmz6c: failed starting cluster creation \"ea-listdnmz6c\" in resourcegroup=\"ea-list-2tprnprxzp8c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-2tprnprxzp8c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listdnmz6c\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2d25236ce7deaa43bc81a6dbd02ee9489e4c7a585960006848bde840c0b39e54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"77100504840186e9b5fee4b7540e7daea129e844b89dc8b99d371650c4f68531","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c34180\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c2vqb28n2z6s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c2vqb28n2z6s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c34180\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c2vqb28n2z6s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c2vqb28n2z6s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2e863a37aaf037ccf25621b598755547a936c73a0acadf7e897fcc86db710d81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"319cb259eb78c0d1bf8cdf9d60b591524a4d613b7fa239d634ac6dac397f50f6","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a982d0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gwwmf6gjgf55/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a982d0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gwwmf6gjgf55/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"35ecdd33aff331c3f2643e0f1091f26f9ddc2ff74e75ffef2492c6283c708487","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"9ebaebedfaf6975ff95a4970a8f1bff72b03b8677d3a8aed73410a5bdba9d566","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-lmjw65\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6a20\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-lmjw65: failed starting cluster creation \"cluster-zstream-4-22-lmjw65\" in resourcegroup=\"rg-zstream-upgrade-4-22-79tsm6r6kxcv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-79tsm6r6kxcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-lmjw65\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-lmjw65\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa6a20\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-lmjw65: failed starting cluster creation \"cluster-zstream-4-22-lmjw65\" in resourcegroup=\"rg-zstream-upgrade-4-22-79tsm6r6kxcv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-79tsm6r6kxcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-lmjw65\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3821476f886ed9fae0810e2950e298b55a29d8b8e44b275a00930864b6e9ec68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"fc9cf6cd8f627b96267ce3a2d2df20ee00f7becd9a6bfddebea3bba60248db33","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ea200\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wc9p6hpc29b6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wc9p6hpc29b6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ea200\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wc9p6hpc29b6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wc9p6hpc29b6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3f4f62cecfd45b5cd181476237371c768f98b2bcd133a991295574b5e5d18c33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"860aff23781cffe752cf788549eb40de12d922321ece7a067ab30debae897099","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062c3e0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9j5894xv2nfk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9j5894xv2nfk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062c3e0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9j5894xv2nfk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9j5894xv2nfk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"426eed758dc902736a640da31fc1bd3ad649a47c1a6c0b293891bd301aceb1d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"91d6b2290275c146a985f2956ee1842c1adbe4cb3327ef55f9f3f2426b0667d9","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992780\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-qq5gvs64ckln\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-qq5gvs64ckln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000992780\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-qq5gvs64ckln\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-qq5gvs64ckln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"46838f5f7957582a22339fed8a9215c63a2f479ca2b4b2f07eff3d065b5e1ff0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"25c4e000866ba7943d0ff38f96d5187f3e10e9481cdf357cce05365f644f12c2","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081aa20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-w2klxbbvrh6r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-w2klxbbvrh6r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081aa20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-w2klxbbvrh6r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-w2klxbbvrh6r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4a57fc96c30109c347883b63d401265780855ccce2f27b9eddb6d3d075682eb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"743a5ab4bbb37dae80df81db4c44a32cb21bb079217ca716a1aef7fb333ab9c5","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4240\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-49bvw77ddhd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-49bvw77ddhd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4240\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-49bvw77ddhd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-49bvw77ddhd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"54e487932234c5aa678fc0648176f8724b8d72890c162207626b430caad95775","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"bb36fc8d69b4ccfcf89bcd6519a5915b66e79f46ab6877afd7421eb5b01710fb","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001310120\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nwsnnfvwzdrd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nwsnnfvwzdrd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001310120\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nwsnnfvwzdrd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nwsnnfvwzdrd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"559b9afb1706a34a7c6799d204f3e2041f1c557f78fbdc18c7da29410fecf435","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"b6e080a5fbaf018893bc01e662e627acbbdae0f7292faf237f307e3db8105b47","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t6w7kn with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001586240\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t6w7kn: failed starting cluster creation \"np-version-upgrade-cluster-t6w7kn\" in resourcegroup=\"rg-np-version-upgrade-t6w7kn-n527hph4clhc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t6w7kn-n527hph4clhc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t6w7kn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t6w7kn with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001586240\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t6w7kn: failed starting cluster creation \"np-version-upgrade-cluster-t6w7kn\" in resourcegroup=\"rg-np-version-upgrade-t6w7kn-n527hph4clhc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t6w7kn-n527hph4clhc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t6w7kn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"55c20736bc7eb169b9824f8db6d735a433faf7aca5803240f2207feebfa29cdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"d00bb74f4e653fe5a51ef79bdc35230c19f456d7de25ee574b5724ead42ab6f9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-zprvbm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d2fa0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-zprvbm: failed starting cluster creation \"cp-ystream-upgrade-4-23-zprvbm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nc75hl29zdtj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-nc75hl29zdtj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-zprvbm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-zprvbm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d2fa0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-zprvbm: failed starting cluster creation \"cp-ystream-upgrade-4-23-zprvbm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-nc75hl29zdtj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-nc75hl29zdtj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-zprvbm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"570089f2918c94524e4f2d7bcf2cd630c72d8dd63ce7758ee9a7bef68d69cbbe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"933c56da08d65cda2aa6845c5dc6f3d13c9baf8e9bf94638891a75c18cc282db","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d385c0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-b2v8h4lk7mr9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b2v8h4lk7mr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d385c0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-b2v8h4lk7mr9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b2v8h4lk7mr9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5ad2ab4fce9b39aebc6e6033c59f01424512cae7c9bf0cd4f16c8dc81d677880","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"a1b4c80ff80950b4b4b75a12d415c5927a2f10d62f1b79a4d8901849717eafe4","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00143a1e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-m7td24fcmrl7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-m7td24fcmrl7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00143a1e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-m7td24fcmrl7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-m7td24fcmrl7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5c4dd743fc8f48236425b67b94eb57139a52f770228b03fabdba3b0104d90bc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"2e22a4a575095366d9288929a006514acfaf8691e9e3d7d3e79cb23cd3ab9b93","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000506300\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-8dfwnhhtxdp6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8dfwnhhtxdp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000506300\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-8dfwnhhtxdp6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8dfwnhhtxdp6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"60b1ba7ab5c1e075b7d077351268d9aca4acfde8625f08f7fe424c393d357bbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"75ef17175d53859e312f86a0739d2f6b44c2045f9da4b4c135abca826451b8ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-6tlfjxd6xqc6/cluster-candidate-4-22-xvfpvx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fba3a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-xvfpvx: failed starting cluster creation \"cluster-candidate-4-22-xvfpvx\" in resourcegroup=\"rg-candidate-4-22-6tlfjxd6xqc6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-6tlfjxd6xqc6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-xvfpvx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-6tlfjxd6xqc6/cluster-candidate-4-22-xvfpvx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fba3a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-xvfpvx: failed starting cluster creation \"cluster-candidate-4-22-xvfpvx\" in resourcegroup=\"rg-candidate-4-22-6tlfjxd6xqc6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-6tlfjxd6xqc6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-xvfpvx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"70069cd2b165196642abf77d191169a6ba094c3ddf6da2711050662796fc2de8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"96b255a8cd6cf44fdebf8d73507be6fe889f89f698f8b5be8e30089737a7dc45","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123c100\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-zhdp8544srss\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-zhdp8544srss/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123c100\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-zhdp8544srss\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-zhdp8544srss/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"71653768535193e2284dd5dd427c0a7bb10571927fc2482a9ab5fb560981df6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"681de8d8b7da047de7080e050ef511cddcd03a883076ac9fc82e103e6deffdcc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-66tjf2\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011c0240\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-66tjf2: failed starting cluster creation \"cp-ystream-upgrade-4-21-66tjf2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-fp5kzmg9k2qb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-fp5kzmg9k2qb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-66tjf2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-66tjf2\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011c0240\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-66tjf2: failed starting cluster creation \"cp-ystream-upgrade-4-21-66tjf2\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-fp5kzmg9k2qb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-fp5kzmg9k2qb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-66tjf2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"74bfccf01d3909c1a53128d29647994735066220c09d22f27f8464acc6562011","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"274d98c050c02ba40012737f76ea63998befa2394a42cec617bc7a856ad7682a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-f8mhn9\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f900e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-f8mhn9: failed starting cluster creation \"cp-ystream-upgrade-5-0-f8mhn9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mf5jqrbn6srb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-mf5jqrbn6srb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-f8mhn9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-f8mhn9\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f900e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-f8mhn9: failed starting cluster creation \"cp-ystream-upgrade-5-0-f8mhn9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mf5jqrbn6srb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-mf5jqrbn6srb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-f8mhn9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7a2d692c834309fe63f70a01f2c8c48b879d1e791b9ca8fae88e0f65c9b12677","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"1861c336e850ec720d54ed12538396a5968d46ac38640e35bbd2462f646633a5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-kf4fc8\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kf4fc8: failed starting cluster creation \"cluster-zstream-4-20-kf4fc8\" in resourcegroup=\"rg-zstream-upgrade-4-20-24x8x7qddc4f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-24x8x7qddc4f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-kf4fc8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-kf4fc8\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e3e620\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-kf4fc8: failed starting cluster creation \"cluster-zstream-4-20-kf4fc8\" in resourcegroup=\"rg-zstream-upgrade-4-20-24x8x7qddc4f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-24x8x7qddc4f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-kf4fc8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7c453db715282f8ffde7649947fd0d372f0fae1261830e1f76fbd670e0a4e911","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"c4cbde7cb1452b4cf73b0cd9d594e432b97d6987d73cde5510ab0cc87df2fec5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-23-hqj25xm5xjpp/cluster-candidate-4-23-x87dzg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010300\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-x87dzg: failed starting cluster creation \"cluster-candidate-4-23-x87dzg\" in resourcegroup=\"rg-candidate-4-23-hqj25xm5xjpp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-hqj25xm5xjpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-x87dzg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-23-hqj25xm5xjpp/cluster-candidate-4-23-x87dzg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010300\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-x87dzg: failed starting cluster creation \"cluster-candidate-4-23-x87dzg\" in resourcegroup=\"rg-candidate-4-23-hqj25xm5xjpp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-hqj25xm5xjpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-x87dzg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8402c303e04eb424916b5ea1014838ddd183a8bbc1346bf44672190c5835615e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"cab56619bdf677848c93b5b8420ef2c26d700e1764280f4910b1d4ffa64fd926","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008123c0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-49kfk2h5cd4x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-49kfk2h5cd4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008123c0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-49kfk2h5cd4x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-49kfk2h5cd4x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8430e3baf67e6b53b093e14fb69e88a2c92bf7a753df9415aea5a0103a0c1833","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"03b6a7c43565f4d6457ac2a12a167a7c246d083a848784af01b179d01a1b1959","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f42380\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-744mhb4wmcsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-744mhb4wmcsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f42380\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-744mhb4wmcsp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-744mhb4wmcsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"870cb01b85464084638b2342a421adb33f57af35107966ef528ff4b31d155cc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"6c40a656c38a297b57950efbb351560b8611e7673ba986f528cd6c276410f7a0","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2120\u003e: \n    failed to create HCP cluster np-dg-minor-5xtqzm: failed starting cluster creation \"np-dg-minor-5xtqzm\" in resourcegroup=\"rg-np-dg-minor-5xtqzm-28nz5th2j6m7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-5xtqzm-28nz5th2j6m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-5xtqzm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2120\u003e: \n    failed to create HCP cluster np-dg-minor-5xtqzm: failed starting cluster creation \"np-dg-minor-5xtqzm\" in resourcegroup=\"rg-np-dg-minor-5xtqzm-28nz5th2j6m7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-5xtqzm-28nz5th2j6m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-5xtqzm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"880b31b7ebaf4f2a00fee18bf944dc823e6bf8359bb4d4d4ed8fbaf1ec6dca77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"4983315498d70cb31e76f7699a313650db35ece01cd779a279712dcb8442887b","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9c260\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wftkcwprcffb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-wftkcwprcffb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9c260\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wftkcwprcffb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-wftkcwprcffb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8d01a4b1eb6c87f531fb833eb08486c979b8123cc0b8579834473eb14dfc44a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"deaac275b010033e9f5d3593bc261037d301f4d2a1bc4467c6ed5a8265146456","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa6340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-89xxsz542trh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-89xxsz542trh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa6340\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-89xxsz542trh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-89xxsz542trh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8d2ed0763c6eccb67e500487fff7c6d01159e9d626bfa7013690358ccdd5dad9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"b5cc2d0a8fce4dd78b8efcb7b33bfb25bfc5ca2e9fa45fefeb729b74cc3d9a91","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000788a60\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-skgrd5ch2wdr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-skgrd5ch2wdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000788a60\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-skgrd5ch2wdr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-skgrd5ch2wdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8ee6c17f35b3c41238cc3a63e019287028db7407e96cf78b8f38525877b32c79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"c89b3ce1fd98fb962801441940537ea6f73edd87b76bd04625e9c99544707285","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-jcpfwd at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001808280\u003e: \n    failed to create HCP cluster np-skip-minor-jcpfwd: failed starting cluster creation \"np-skip-minor-jcpfwd\" in resourcegroup=\"rg-np-skip-minor-jcpfwd-gvqsdxzjh8vp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jcpfwd-gvqsdxzjh8vp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-jcpfwd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-jcpfwd at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001808280\u003e: \n    failed to create HCP cluster np-skip-minor-jcpfwd: failed starting cluster creation \"np-skip-minor-jcpfwd\" in resourcegroup=\"rg-np-skip-minor-jcpfwd-gvqsdxzjh8vp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jcpfwd-gvqsdxzjh8vp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-jcpfwd\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"980cba5d8496acafcbdf2b195ae9420355a1d1ad18cbd5232b0f54b23d2bcfe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"e913588f2e94836d7f113c48b7e55eb49e70db7bd51ad09a6a1f974c6f20970f","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00154a810\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00154a810\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9f5e529f6fa018c05a1f2fc450ce2c642d9c1eaf96b678c22a56f7694e268e28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"97788927dec662d1d0e17e084bc25e4afd04ae188871b50b8331a02a8e1512be","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014940e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-2pqz77p5k645\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-2pqz77p5k645/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014940e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-2pqz77p5k645\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-2pqz77p5k645/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a5114020a437b9eb715c67a514b6c982a103ad8e0ce4e43c692a431b4abbc53e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"56aa8b7d41bd143dfe4120e95fd6d7e4c9bbd66a4330de0e627de46d6435247b","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614500\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hwvkqfn2c69n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwvkqfn2c69n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614500\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-hwvkqfn2c69n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwvkqfn2c69n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a5ed4d2d6a1ec9c0309226bdb7c01e7b02cc0f93e95deb77c11b52fca6b46dce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"4ae18b7f4aa0dc6f2688473720e3fac39e9593abc7a2693fad640b4439398de5","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002280\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-sp7x4pd4zkrt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-sp7x4pd4zkrt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002280\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-sp7x4pd4zkrt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-sp7x4pd4zkrt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ae0d3ef835cace312a75586acdba118872bce5f80aa64165048edaa966a59277","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"112783ba7b95c403dd3d93efc5449c10448c7bec0d1079842d6a9d439cbc44d9","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-7pqfcm with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134e120\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7pqfcm: failed starting cluster creation \"np-version-upgrade-cluster-7pqfcm\" in resourcegroup=\"rg-np-version-upgrade-7pqfcm-2zqnz4g9wfx5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pqfcm-2zqnz4g9wfx5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pqfcm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-7pqfcm with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134e120\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7pqfcm: failed starting cluster creation \"np-version-upgrade-cluster-7pqfcm\" in resourcegroup=\"rg-np-version-upgrade-7pqfcm-2zqnz4g9wfx5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pqfcm-2zqnz4g9wfx5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pqfcm\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b2092c899d3b57735c49151b9fb75a3ebbbb8af2244e0d485d33d6fae2780d9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"f036658b7e69c80c317d2d24a945358af0e8f394a6c34d18d961e843159d8f04","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-nr97qpqbnq89/cluster-candidate-4-21-vbtv2b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000914640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vbtv2b: failed starting cluster creation \"cluster-candidate-4-21-vbtv2b\" in resourcegroup=\"rg-candidate-4-21-nr97qpqbnq89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-nr97qpqbnq89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vbtv2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-nr97qpqbnq89/cluster-candidate-4-21-vbtv2b should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000914640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vbtv2b: failed starting cluster creation \"cluster-candidate-4-21-vbtv2b\" in resourcegroup=\"rg-candidate-4-21-nr97qpqbnq89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-nr97qpqbnq89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-vbtv2b\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b2152a1bfff987272eb7800a69df6221df51acabeb2318575513edc2ba12e614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"ca6b9a69cdcb8d678a64cc522c2eded46cc7048e98db8b60701be431f4451559","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a49080\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-grmp56x8mfgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-grmp56x8mfgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a49080\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-grmp56x8mfgq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-grmp56x8mfgq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b7c5bd724267f0025362afcb11aacbbb8190e93dbf93431fd1a69b30586499e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"aaf98359994aa5a99a180a2d9a07c8e60eeb104fc6de4939bffabcaed04533c6","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca01e0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-lzr7ggchsgl8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lzr7ggchsgl8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca01e0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-lzr7ggchsgl8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-lzr7ggchsgl8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c14602758bdf310ddf2146684092aae26b6c3a5f924f7f12b1d9015f66a35f83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"8637b2f6ba5650241aaaaa116a2282cef63f77b6b74df2a63cb50451ef5af63d","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098c6c0\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dzql26rplxsd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-dzql26rplxsd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098c6c0\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dzql26rplxsd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-dzql26rplxsd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ca6d1fd5012c6dab36034fa8da99f371320ce358861a115c800d6bf31af2cfe0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"bdb9550e9115080d784a804bd1ded215f1f23126748ec7f4f9a2e0e56e195c67","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048260\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-n85wjx2ql25d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-n85wjx2ql25d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048260\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-n85wjx2ql25d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-n85wjx2ql25d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cb17b9495eedd159e1aea1d7254963ab7543aca3c002211093b33d8bcb0a0423","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"e52bb7c8b8702e4f5f0f12d35371440fa26f8e7305b750f17f3780409d1cea4b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-7pqq8c\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2cd60\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-7pqq8c: failed starting cluster creation \"cluster-zstream-5-0-7pqq8c\" in resourcegroup=\"rg-zstream-upgrade-5-0-psl5tsd9krt8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-psl5tsd9krt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-7pqq8c\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-7pqq8c\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2cd60\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-7pqq8c: failed starting cluster creation \"cluster-zstream-5-0-7pqq8c\" in resourcegroup=\"rg-zstream-upgrade-5-0-psl5tsd9krt8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-psl5tsd9krt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-7pqq8c\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d2a49b7176409d7eacdbbcbb6126173b259917d8369f3bdbab28c643b0b9fb6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"90e2f9b7aba95bedd6261d87467024ea908cc72b001a451c8f70266a78eb279a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006871a0\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-npl4lqlc2j28\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-npl4lqlc2j28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006871a0\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-npl4lqlc2j28\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-npl4lqlc2j28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d4649ac508949ac74f454d8ce706ad9c1e1cf27bde8a6026c66cb972fe6a8606","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"11d8be1ddc0e0a10220e86111161442393e8e4fe9d7bc0f56690967982734e25","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe01e0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-k5wx2hsgkgzp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-k5wx2hsgkgzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe01e0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-k5wx2hsgkgzp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-k5wx2hsgkgzp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e213c19d3bf058a09285efd65f34bb276719bebde16ceaac7184b3bd25a7145e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"a70aa82f421e4acdb041ad16496c250af687badd0c3ab467c9c3a946de950fc2","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-24kkdn\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000789c60\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-24kkdn: failed starting cluster creation \"cp-ystream-upgrade-4-22-24kkdn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-jchspf48txds\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-jchspf48txds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-24kkdn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-24kkdn\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000789c60\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-24kkdn: failed starting cluster creation \"cp-ystream-upgrade-4-22-24kkdn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-jchspf48txds\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-jchspf48txds/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-24kkdn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e3735d6bddd2f0ebe4665271e214c7026a56f38d41402b76cec3ea369dbc21cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"10750c53b0c1dc3c380bfb98ac1ab6ed054bc9fb7563fe02eae47c73ddc7789a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-rfcvwdhs2s8z/cluster-candidate-4-20-fb65qk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000911220\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-fb65qk: failed starting cluster creation \"cluster-candidate-4-20-fb65qk\" in resourcegroup=\"rg-candidate-4-20-rfcvwdhs2s8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rfcvwdhs2s8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-fb65qk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-rfcvwdhs2s8z/cluster-candidate-4-20-fb65qk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000911220\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-fb65qk: failed starting cluster creation \"cluster-candidate-4-20-fb65qk\" in resourcegroup=\"rg-candidate-4-20-rfcvwdhs2s8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rfcvwdhs2s8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-fb65qk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e55739cdc668af884b359f5b6bb760b9615c686e41700728e799d1563eae3348","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"08271482dccdc39bddd2cb376b98b288f0d159b3c06e543fde79c1ed7f9713d0","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-5m6fvn\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106c220\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-5m6fvn: failed starting cluster creation \"cluster-zstream-4-21-5m6fvn\" in resourcegroup=\"rg-zstream-upgrade-4-21-585ggz8lqzc9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-585ggz8lqzc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5m6fvn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-5m6fvn\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106c220\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-5m6fvn: failed starting cluster creation \"cluster-zstream-4-21-5m6fvn\" in resourcegroup=\"rg-zstream-upgrade-4-21-585ggz8lqzc9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-585ggz8lqzc9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5m6fvn\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e650531628b9f8a6230b67cdd4fcb0e81125936f7054c175ca0008fb0ea68653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"53e43e1b5eb0e6100a4da856364350993487635e225b11af657e0f6c38671ad0","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd01a0\u003e: \n    failed to create HCP cluster np-downgrade-j4m7th: failed starting cluster creation \"np-downgrade-j4m7th\" in resourcegroup=\"rg-np-downgrade-j4m7th-k6x7c5vhdzqw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-j4m7th-k6x7c5vhdzqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-j4m7th\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd01a0\u003e: \n    failed to create HCP cluster np-downgrade-j4m7th: failed starting cluster creation \"np-downgrade-j4m7th\" in resourcegroup=\"rg-np-downgrade-j4m7th-k6x7c5vhdzqw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-j4m7th-k6x7c5vhdzqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-j4m7th\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e89ca255ff13e10ca0fc75a6f041673b846794ceac41bbe5dd5b0fc2adb96f1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"d2bb6e2d403656c49ab04e83c3bb7e2ff4b986e11e9b4058682e200cf346b8b7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001690100\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxsb4d4dcndm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mxsb4d4dcndm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001690100\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxsb4d4dcndm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mxsb4d4dcndm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"eca2104dfbd1dc53f28b4dd311eb468ab3dd857175a8a8ef37142f77b52298f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"d2c0b34854603aac3e86bd85677fb864f529bab16a93499e7d50cd286c96374b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946140\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-gxzd4bqsq7ff\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-gxzd4bqsq7ff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946140\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-gxzd4bqsq7ff\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-gxzd4bqsq7ff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f16ead6a938760c27e405656d60f772c3649d0011efaabda3c824c8f96ecea74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"c13d02c7d475af09cbae9d547b5a17ff9551e2bd6740d79a99ba3a7e492e2feb","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001044400\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8g9xp5jpnqwl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8g9xp5jpnqwl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001044400\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8g9xp5jpnqwl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8g9xp5jpnqwl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f3ab442c775e913ee4bdaa6e03dbda6b9603c3741187ea08fac1a174e2151a85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"74c7a39324f13af264582228997f17104143bff25f38bbc535fb0f57457f5572","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cde9a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7gkqvtjn5hrh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7gkqvtjn5hrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cde9a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7gkqvtjn5hrh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7gkqvtjn5hrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f4b18a5178fff8993601be955170a9881098b6e1f40509848917f654c8d31ba8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"25fa332b34d3d0c6a3770683fdb0f70e3a425bd1ddea7417e681b87d353bf917","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-qd9tq7\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d82360\u003e: \n    failed to create HCP cluster np-noedge-qd9tq7: failed starting cluster creation \"np-noedge-qd9tq7\" in resourcegroup=\"rg-np-noedge-qd9tq7-j6nd68sppgnf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qd9tq7-j6nd68sppgnf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-qd9tq7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-qd9tq7\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d82360\u003e: \n    failed to create HCP cluster np-noedge-qd9tq7: failed starting cluster creation \"np-noedge-qd9tq7\" in resourcegroup=\"rg-np-noedge-qd9tq7-j6nd68sppgnf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qd9tq7-j6nd68sppgnf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-qd9tq7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fcaa72a9a500563086621a5accd1a02e25675eb51d89d490596be29b221868e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"0991f03a2ec81a59452280bcd4e0f4b5e4ae53001ecb34846994d6f30417c74b","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92300\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zn5n8g9qjq6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-zn5n8g9qjq6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92300\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zn5n8g9qjq6p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-zn5n8g9qjq6p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ff75470caf9f6ff9b13369e8a94201cab86408fcbda18b925c669be961b5b499","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102810351396458496","occurred_at":"2026-09-23T17:20:30Z","failed_at":"e2e","signature_id":"e7aa44e9bbee4b315fd192b4e7f53f081c938e5e5502ae7ad2f73d753bd1addc","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009582a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gggk2qcjxcn8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gggk2qcjxcn8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009582a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gggk2qcjxcn8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gggk2qcjxcn8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":58,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":58,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T02:00:41Z","started_at":"2026-09-23T02:00:41Z","completed_at":"2026-09-23T02:20:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:34:36.297931734Z","timing_metadata_checked_at":"2026-09-23T02:34:36.297931734Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:34:36.301155423Z","region_metadata_checked_at":"2026-09-23T02:34:36.301155423Z"},"failed_at":["e2e"],"failed_test_count":56,"occurrences":[{"row_id":"0102ccee1b2e54d8ed2e26df04bc59c66e681aba1434e8beffb1070ca94affaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"d0d4aed9b08ccbcf282b2f7c430431f065a7afdf35b7962e9f9e39791686a77b","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57840\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-j4n5xfxkstwg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-j4n5xfxkstwg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57840\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-j4n5xfxkstwg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-j4n5xfxkstwg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"014e7eeee675fcd97715f55bee3df58d6ba09d28b5c73028486c86b98793dbc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"f3a544a747d524e7c42a166bea6da176cfc9fd44cafd865f6ef95fd355051d3b","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005dc160\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-22256vxjwxzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-22256vxjwxzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005dc160\u003e: \n    failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-22256vxjwxzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-22256vxjwxzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0269816fa23d3d400fbaaed0b76a9c58458b237ad817edb282aff8419e24c125","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"5a6930dc2b2b119484f09428ee05bc3e9aecd5bc4212a7db9f2b1c6cb97736a7","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a4440\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-g2r7df4nbvtl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-g2r7df4nbvtl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a4440\u003e: \n    failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-g2r7df4nbvtl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-g2r7df4nbvtl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"083e1b73b9c7ac1d430afceff0b06ebe9b5d34bea68b04d320da934970907b70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"f8270308b8276ecf0eab70cb17d92106639dd7da4b191567abdb0119310f33b7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005886e0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tnkkshhqpgpc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tnkkshhqpgpc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005886e0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tnkkshhqpgpc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-tnkkshhqpgpc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0ca23776a98e11acc429a6a502da1e99a2333a57987e20c59b146b070b7c4725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"fb2833678dadd3afaa201320b7355bd70292569e1299718dce7b9367cb5b7866","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ef1a0\u003e: \n    failed to create HCP cluster ea-listmfk6rl: failed starting cluster creation \"ea-listmfk6rl\" in resourcegroup=\"ea-list-bkxpqlfv4c6z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-bkxpqlfv4c6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmfk6rl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ef1a0\u003e: \n    failed to create HCP cluster ea-listmfk6rl: failed starting cluster creation \"ea-listmfk6rl\" in resourcegroup=\"ea-list-bkxpqlfv4c6z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-bkxpqlfv4c6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmfk6rl\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0d73a2a2ccb184b96f2a6ac7092b83f1ddecf16d0d80a81e7f6202186857f026","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"2151fe66fba30fc7b88c2c51c44e694bfe080433a2c89a3e8c0231bf9e722eaf","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a2500\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-fwgpzzwmqdsk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fwgpzzwmqdsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a2500\u003e: \n    failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-fwgpzzwmqdsk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-fwgpzzwmqdsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0ec957d9e27ffa83d397b392a24d2a93e0ecb57f9361cf57df777d54476c8520","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"11d5639b580236cb0256012436dce0eb857fee45e4e28bbc667bf91a1e2a0356","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00067a300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-rsb2d87wnkmj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-rsb2d87wnkmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00067a300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-rsb2d87wnkmj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-rsb2d87wnkmj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"10ada191e1f39c26c174c7ccc7bb31d3c181439fac35cf367a0e87a85e02bc45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"cac1cd023c3da045c4f4887f2d690e9268981477403935ba03e92dca7a280d1d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-wcfsw2 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184a100\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-wcfsw2: failed starting cluster creation \"np-version-upgrade-cluster-wcfsw2\" in resourcegroup=\"rg-np-version-upgrade-wcfsw2-tgvfvwtvq9vl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-wcfsw2-tgvfvwtvq9vl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-wcfsw2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-wcfsw2 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00184a100\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-wcfsw2: failed starting cluster creation \"np-version-upgrade-cluster-wcfsw2\" in resourcegroup=\"rg-np-version-upgrade-wcfsw2-tgvfvwtvq9vl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-wcfsw2-tgvfvwtvq9vl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-wcfsw2\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1756342668a6acc63bd71007aaa37b11c465c149724b6c59b20d51cb3f772d7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"5c4cc073f23d1d65b6648dcd7501f81e8e79d5adb2e4adf87d81a8923986aac3","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec4100\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5zhpbqbl9m9q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5zhpbqbl9m9q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec4100\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5zhpbqbl9m9q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5zhpbqbl9m9q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1afff9c083839e589db2ace357aa3d0b4052df24cd8e8e184b77fa76c05934af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"fdc3dd15ff2799efe3a6bd04abb34c603047b1aaf5a9d25deb018df7ea20395c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-h89dm9lhsl2f/cluster-candidate-4-22-w9djqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a1a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-w9djqt: failed starting cluster creation \"cluster-candidate-4-22-w9djqt\" in resourcegroup=\"rg-candidate-4-22-h89dm9lhsl2f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-h89dm9lhsl2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-w9djqt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-h89dm9lhsl2f/cluster-candidate-4-22-w9djqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a1a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-w9djqt: failed starting cluster creation \"cluster-candidate-4-22-w9djqt\" in resourcegroup=\"rg-candidate-4-22-h89dm9lhsl2f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-h89dm9lhsl2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-w9djqt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1d4a984a161d7d269e9b90b9c580dec159cbfdc1f9dd9c07dee08dc5dcf528ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"289eec3a6e696ef46ddf1c7f86dfde24b915efbf8c0d349295b1e0b4238e5bb7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-95hssq\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24480\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-95hssq: failed starting cluster creation \"cluster-zstream-4-22-95hssq\" in resourcegroup=\"rg-zstream-upgrade-4-22-mw82tvb5wt8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-mw82tvb5wt8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95hssq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-95hssq\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24480\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-95hssq: failed starting cluster creation \"cluster-zstream-4-22-95hssq\" in resourcegroup=\"rg-zstream-upgrade-4-22-mw82tvb5wt8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-mw82tvb5wt8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95hssq\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"22cfa26368137eeadb89a5f3de66d741d98cec4c8f0da59620ffa1c047a12e67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"55745c23236c64509e7a4aa08f812f9a02fb58ccdc1c1d4ecfbcb51cf8a27003","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e12740\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-fnhqkgb28q5j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-fnhqkgb28q5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e12740\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-fnhqkgb28q5j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-fnhqkgb28q5j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2549959cc2b68c4cea3b59bcb709a9c8c78f042b6ecb961495a359c8f4869d2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"7a99d6719a1f3f880e142b01cf1409fe5b22d96c4ae9d411a7727111fa56ca3f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a841e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2pscx6lt6hbw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2pscx6lt6hbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a841e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2pscx6lt6hbw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2pscx6lt6hbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2dce8b17cf137ac4c8b22a34435c93c38ba0e0e93c9272eea8490d77a3c9b52d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"3e647150f33e3ffc56d2c1571c807f82cff11cc470178b3156913149e09c5fa8","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94200\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qhgbvbhnhp7c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qhgbvbhnhp7c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c94200\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-qhgbvbhnhp7c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qhgbvbhnhp7c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"30d1cb70ff0a83444c79e21a841c31a14c5da057e885c07978502be45f4c90ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"abd5b0bde20929e08f3cff92d0e75df98245e589a0b828e32c756457f0299f12","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c00780\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7q6zb9qhrlww\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-7q6zb9qhrlww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c00780\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7q6zb9qhrlww\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-7q6zb9qhrlww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3430c69f7085e55f300f761c898019b7b70e812b23fd8f4d370656373f45fb70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"8b94f2133755206c3dd7834edbf78fd4d7dc593a42786d5b82b69feeefc4db0c","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001026160\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-s8j9mmmntb7s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-s8j9mmmntb7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001026160\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-s8j9mmmntb7s\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-s8j9mmmntb7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"343ffff7fc37c57997b5295b401940ffe30326dc9698faef48cc1af25c47c17a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"8d905c196618d4530f22d1430069e41b0f742c52c337188e2d26d0f1afe42e3b","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084b4c0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-vjfmvmg6928d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-vjfmvmg6928d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084b4c0\u003e: \n    failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-vjfmvmg6928d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-vjfmvmg6928d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"364acadfed18af44c0ac4847c68fc77786631f01a470e934a95a345a033a638c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"17df3d13f5564cb9f511516ded4286c659befbe31426e6104494fef23a28c7dc","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012280\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4lj84nw2x245\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4lj84nw2x245/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012280\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4lj84nw2x245\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4lj84nw2x245/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b90933b7e58eabcee6805801d9c79e95609213b84e159a713cc02ab63ad68e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"73468874c7c9cf8732ad731b3ea78e3133f91cd177d1caf52e6b177a524605a6","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001380240\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8xv96lzbm5cx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8xv96lzbm5cx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001380240\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8xv96lzbm5cx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8xv96lzbm5cx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"42555792220f902865792cf4f6c7d81d17fd126a83d3938903412cc1a27f1146","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"a016b5b38d2a4d75d53b6bac598237104a6247cc3099e70e2994bd0e3e4f3c67","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090d780\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-4znnjj2bvmhl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-4znnjj2bvmhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00090d780\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-4znnjj2bvmhl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-4znnjj2bvmhl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c827e8b27c3dcff537220a0af2f1a4a9e950f8f3e9705851e7dd8a76ec23333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"b4a5a50af6009581dc90fafaa87f4ddba1bd3e00165b7e289232ffb8b1070898","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079e0e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-lpzbtxsqnsjm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lpzbtxsqnsjm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079e0e0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-lpzbtxsqnsjm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lpzbtxsqnsjm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c9868594c5c6f6116f7e780bf23eed58ce45ad53f32e9d5044b35e2e30a51f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"89bd7a623adb5354a5f8c65504e30ec84efdd85ed2a46a72c2401b213c912802","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a2120\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fpsrssl7cjkj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-fpsrssl7cjkj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a2120\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fpsrssl7cjkj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-fpsrssl7cjkj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"555ffbc7350a3e799bf34f79ca3108a7dc77bb8e4f5a909341237931238f6ae1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"56826cf4f16e45e7487b87b2af3d3c6ae3a0e3cfa1f3cc7fa5babe12c95f8be4","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02640\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-svfw267gk9r2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-svfw267gk9r2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02640\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-svfw267gk9r2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-svfw267gk9r2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5a750a2ab9e482f241831cad7ecf1472e3890e045ca1e67f9e6cd4a4713e99e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"a47196edb9e16f5059f4721a7f42f6ff2b4d49de1c50e00267532d5dcea13e25","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b66340\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-q4smrwpc7lth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-q4smrwpc7lth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b66340\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-q4smrwpc7lth\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-q4smrwpc7lth/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"660c1d7bb99655b4c9d815fc4cece738348f30ce2b418487a9b22bf97d0f99bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"14a5ec07af57bb86a839a52a1b638f1fd866b70c11189aedcf0432ac10bfe09b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-6zw86fclhnhh/cluster-candidate-4-20-7rdtb7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff62a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7rdtb7: failed starting cluster creation \"cluster-candidate-4-20-7rdtb7\" in resourcegroup=\"rg-candidate-4-20-6zw86fclhnhh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6zw86fclhnhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-7rdtb7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-6zw86fclhnhh/cluster-candidate-4-20-7rdtb7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff62a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7rdtb7: failed starting cluster creation \"cluster-candidate-4-20-7rdtb7\" in resourcegroup=\"rg-candidate-4-20-6zw86fclhnhh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6zw86fclhnhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-7rdtb7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6968f74f104d473794bcc21337e5076d87027e0c949fd0e22f224970dbb529f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"a6772d6c106905eb9f6a3066eec528ee5954761eb637001321b4f9f88fd5856b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-gk8mbp\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5e380\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gk8mbp: failed starting cluster creation \"cluster-zstream-4-21-gk8mbp\" in resourcegroup=\"rg-zstream-upgrade-4-21-k25gftsdpj5n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-k25gftsdpj5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk8mbp\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-gk8mbp\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5e380\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gk8mbp: failed starting cluster creation \"cluster-zstream-4-21-gk8mbp\" in resourcegroup=\"rg-zstream-upgrade-4-21-k25gftsdpj5n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-k25gftsdpj5n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gk8mbp\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6a7f8df8a692bae8175854ddf4f35da9ebadbfd2b6179182318e39a1e173f71b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"293200cbe5510f7bd2d0f01fa85b6491a31df5a39073a66ae7eb7c5c4fada6ea","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d32140\u003e: \n    failed to create HCP cluster np-dg-minor-89j949: failed starting cluster creation \"np-dg-minor-89j949\" in resourcegroup=\"rg-np-dg-minor-89j949-ftfm8c66f846\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-89j949-ftfm8c66f846/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-89j949\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d32140\u003e: \n    failed to create HCP cluster np-dg-minor-89j949: failed starting cluster creation \"np-dg-minor-89j949\" in resourcegroup=\"rg-np-dg-minor-89j949-ftfm8c66f846\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-89j949-ftfm8c66f846/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-89j949\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6bdf35a034695290d1ffebff3c9088e75896d0b1db35e88ab352fa391b840a54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"05e00ea7e8fca29e91df7921854d0045d7d268a32fd5b1fb898fc496d1176180","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028ba100\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-np755vjcrftf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-np755vjcrftf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028ba100\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-np755vjcrftf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-np755vjcrftf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6f61262c5556c01cee5cf536afd131839d8f83ffcb5c6d7051e6e85441792f69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"60b4f8f14ba59be897db2077b4c610fcc0496c981276a321ae544895b832ed97","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028b02a0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-rrgpx55nq5gt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rrgpx55nq5gt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028b02a0\u003e: \n    failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-rrgpx55nq5gt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rrgpx55nq5gt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"706a75a12c44fdd6c1d93b2f8e0c00691caef45bf1947d18f66968ffef205ab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"e5745b7eb179164bba7685d4463315e3b097f4a638a982a2fd442bba90e4bc17","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e5e360\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-79r5dzshjfjd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-79r5dzshjfjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e5e360\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-79r5dzshjfjd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-79r5dzshjfjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"70d7b828db84fb47ca012ef11926eeab21a5ff620514fdb459a612bebf72c30f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"b0a448cb87c1db6abd793ac1e2397d859df5b317704c9daa67d5658b46c0d87a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001526320\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-xwmd8q84hgkl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-xwmd8q84hgkl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001526320\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-xwmd8q84hgkl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-xwmd8q84hgkl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"75137bd50eb00658788ebb29a911fe5fd681fce465dd84b4dbaa5c5a488a8fb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"61db7bb228bffe819303eb1c357004a6e589362fbc7f05ce28fb3a3517ae14d2","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-bnwsnt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e3040\u003e: \n    failed to create HCP cluster np-noedge-bnwsnt: failed starting cluster creation \"np-noedge-bnwsnt\" in resourcegroup=\"rg-np-noedge-bnwsnt-dgc7qmfcddg5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-bnwsnt-dgc7qmfcddg5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-bnwsnt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-bnwsnt\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e3040\u003e: \n    failed to create HCP cluster np-noedge-bnwsnt: failed starting cluster creation \"np-noedge-bnwsnt\" in resourcegroup=\"rg-np-noedge-bnwsnt-dgc7qmfcddg5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-bnwsnt-dgc7qmfcddg5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-bnwsnt\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7b8290d46f21fb451a5ddb35e89f4118be9078ed45cc69294f05c17237ad4ab4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"4cc6e4155ae2365f53ebd0b1ffad135c6f6a236f7ef22d5a24dbc10df7bdd269","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-9s6lkg\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef5020\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-9s6lkg: failed starting cluster creation \"cluster-zstream-4-20-9s6lkg\" in resourcegroup=\"rg-zstream-upgrade-4-20-42cxz7mjjbj2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-42cxz7mjjbj2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9s6lkg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-9s6lkg\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef5020\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-9s6lkg: failed starting cluster creation \"cluster-zstream-4-20-9s6lkg\" in resourcegroup=\"rg-zstream-upgrade-4-20-42cxz7mjjbj2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-42cxz7mjjbj2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9s6lkg\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7d849f8e567dbe07783e7e432a226fddb3ccccd30c495bb9a72afd489476dc9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"5afa96be6ec41f79a3fe07b706176526c35b8138b5810b57ec811f7943de2343","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-tjdjx7\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c4440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-tjdjx7: failed starting cluster creation \"cluster-zstream-5-0-tjdjx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-vzh67fzgfvck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-vzh67fzgfvck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-tjdjx7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-tjdjx7\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c4440\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-tjdjx7: failed starting cluster creation \"cluster-zstream-5-0-tjdjx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-vzh67fzgfvck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-vzh67fzgfvck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-tjdjx7\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"828e370f1de2300231da45cc5f37458402192dfaf2c729662a609a80ef271fef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"9de186bca5f6094a9f6224b75e41681f8fa5fbeb6c2779f1896ba6003e742fee","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df82e0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6b525qt6f6jj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6b525qt6f6jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df82e0\u003e: \n    failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6b525qt6f6jj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6b525qt6f6jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8ad83eaa3f1594df69bfdbd00830a16fe5092a1346db2ac19ac57cbe74323c4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"2404a21cae14e1faee7ca7136c2409c3fd7e7c4f15c1dc45b8b5f77f87c756de","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014e6140\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-mgsrktnswwmv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-mgsrktnswwmv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014e6140\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-mgsrktnswwmv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-mgsrktnswwmv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"92188c31f1eccce06ba32192b2e7cce4336260a785312796880eb1439b7486ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"8790ec9b27ab1f793c595a203edf5c900eb4a06cb5959f9f40e481ce25b3a61a","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nqjgxv with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d8220\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-nqjgxv: failed starting cluster creation \"np-version-upgrade-cluster-nqjgxv\" in resourcegroup=\"rg-np-version-upgrade-nqjgxv-f2wzh798npkj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nqjgxv-f2wzh798npkj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-nqjgxv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nqjgxv with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008d8220\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-nqjgxv: failed starting cluster creation \"np-version-upgrade-cluster-nqjgxv\" in resourcegroup=\"rg-np-version-upgrade-nqjgxv-f2wzh798npkj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nqjgxv-f2wzh798npkj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-nqjgxv\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"93901a71aeff0e8e7b3ee90b741dedf448f66da9881868a20ee7acaa809d2538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"d09e517a2b65e275aa9848c4e08fda1f72ad388ed8491a5735bb912f6ee48eab","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b19a80\u003e: \n    failed to create HCP cluster np-downgrade-ppzl59: failed starting cluster creation \"np-downgrade-ppzl59\" in resourcegroup=\"rg-np-downgrade-ppzl59-6bp5w8kqrdc5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-ppzl59-6bp5w8kqrdc5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-ppzl59\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b19a80\u003e: \n    failed to create HCP cluster np-downgrade-ppzl59: failed starting cluster creation \"np-downgrade-ppzl59\" in resourcegroup=\"rg-np-downgrade-ppzl59-6bp5w8kqrdc5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-ppzl59-6bp5w8kqrdc5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-ppzl59\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9d3489ac2e89c44b85089a0246ddc8823787f06c88f8081f01e3d3e7220bbbd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"480cbc0b92eb45b732e39ec5305ca8fd33c7a9199ad3f175f39e7864760ec4fd","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-w95bj9 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0c0e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-w95bj9: failed starting cluster creation \"np-version-upgrade-cluster-w95bj9\" in resourcegroup=\"rg-np-version-upgrade-w95bj9-bzlsbjrftkk4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w95bj9-bzlsbjrftkk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w95bj9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-w95bj9 with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0c0e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-w95bj9: failed starting cluster creation \"np-version-upgrade-cluster-w95bj9\" in resourcegroup=\"rg-np-version-upgrade-w95bj9-bzlsbjrftkk4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-w95bj9-bzlsbjrftkk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-w95bj9\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a2f15a3929b3f892e65d0c8accdda1459fce9318a7e50d603165c39a5bee4083","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"ae1744745976b65d4485f1254ff1f7de0714d5bef862787f9789928eb0679c2b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-9rlflwwnznkv/cluster-candidate-4-21-xhlsgk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a52440\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xhlsgk: failed starting cluster creation \"cluster-candidate-4-21-xhlsgk\" in resourcegroup=\"rg-candidate-4-21-9rlflwwnznkv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-9rlflwwnznkv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-xhlsgk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-9rlflwwnznkv/cluster-candidate-4-21-xhlsgk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a52440\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xhlsgk: failed starting cluster creation \"cluster-candidate-4-21-xhlsgk\" in resourcegroup=\"rg-candidate-4-21-9rlflwwnznkv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-9rlflwwnznkv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-xhlsgk\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ab139e069b8f8d416a871819a7b52cdeeb45c0f8ec17ac7722dc4266687174b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"ab318d0dddcdf34758c7bdb0bc2d5ce297d508c1f6dd190886b2c9185e339a90","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-vsjwr4 at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2c700\u003e: \n    failed to create HCP cluster np-skip-minor-vsjwr4: failed starting cluster creation \"np-skip-minor-vsjwr4\" in resourcegroup=\"rg-np-skip-minor-vsjwr4-bsn56prrsfv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vsjwr4-bsn56prrsfv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vsjwr4\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-vsjwr4 at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a2c700\u003e: \n    failed to create HCP cluster np-skip-minor-vsjwr4: failed starting cluster creation \"np-skip-minor-vsjwr4\" in resourcegroup=\"rg-np-skip-minor-vsjwr4-bsn56prrsfv6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vsjwr4-bsn56prrsfv6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vsjwr4\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aee72a42569398f839e9113e5c7611ed9bbe6c424b59f678bd823e49b7c6fa66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"f14e289d6a4ebfe3fb35f46d7570468b025aa7a11dd277e640517937dbc0a918","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f2180\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-hxl9zp97mlgn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hxl9zp97mlgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f2180\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-hxl9zp97mlgn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hxl9zp97mlgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b5079c80aec2833346fb1c96dfd1b6c50ead0841154d12b82ed4e5125c2b3546","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"9168c97144a1d17cd7592e9a7ab07318188e5559433bd9a9095c6df99462ecc6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040140\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-t4djcjjxltjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-t4djcjjxltjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040140\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-t4djcjjxltjt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-t4djcjjxltjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ba73ac37eed89b957d53ec3bbffc48019009d94ad44b856b15f50c32d59083b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"39ce6c36b21ff03c35642644eded4e8649bd345881f2acc5adac6b656f377e15","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tgns2n\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015630c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tgns2n: failed starting cluster creation \"cp-ystream-upgrade-5-0-tgns2n\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rxzl7kxnnt9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-rxzl7kxnnt9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-tgns2n\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tgns2n\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015630c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-tgns2n: failed starting cluster creation \"cp-ystream-upgrade-5-0-tgns2n\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rxzl7kxnnt9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-rxzl7kxnnt9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-tgns2n\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bb20eae7f9cdabf17d8db5c0448bfc4b8917398d646c6993b77710cd227a2895","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"23635ac418e137f892faf70ba7be1206ece8952522444085c8aaa3f5478c3c9f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a38260\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cqdlgmxn4w4v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-cqdlgmxn4w4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a38260\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cqdlgmxn4w4v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-cqdlgmxn4w4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bf8378903417cbe68e149d3f50940cd0daea36862a7c5f0ad336a3578787c5db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"5364033772494115e4b5f5e40cf01432156a12589305922ac4726f437c03dae2","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000564120\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-z7q4w677jrcv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-z7q4w677jrcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000564120\u003e: \n    failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-z7q4w677jrcv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-z7q4w677jrcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c2a7c43d3361068cbe97a886647bd5ceb809f06fa1b65ec70c71aae101ea4349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"5c703ec8433f62e4f8db39dbfd922f3949682c8caf1f3cf28f93dfdf421976dd","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9zmvzjspzdbt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9zmvzjspzdbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9zmvzjspzdbt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9zmvzjspzdbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c990a697477a35aa2cdedbe66479fca198981e44b40ee3ec276f1403e51a0c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"c5e0c7d19a8e2736fe9718b51962cb6ca433e6e88b4677be6a3d840fdf36b0ff","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpgm4r\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018b0120\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-tpgm4r: failed starting cluster creation \"cp-ystream-upgrade-4-22-tpgm4r\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-mgqcx8h7xftx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-mgqcx8h7xftx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tpgm4r\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-tpgm4r\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018b0120\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-tpgm4r: failed starting cluster creation \"cp-ystream-upgrade-4-22-tpgm4r\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-mgqcx8h7xftx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-mgqcx8h7xftx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-tpgm4r\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d4a2ec1234a34d4970dba6d4f15c45a5369a4305bd08a9d99d0cab62fb2ba33b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"4af054ec58b34153ea3601f9333faa1564f06c8e2e087fdd7e3bd3772cfac636","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018261e0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-qh5rmkcjlrnr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018261e0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-qh5rmkcjlrnr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"da1fd5caa8718ed7f03c6461228912694a2e69ae7975e618c1accb7510bb693b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"897c8759b2596cfd877d3de8c8f27dffa79f5a10c27cc01aff9f1f90af11382b","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e5a0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e5a0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"deefd051acc33b694b5ffed812171ddbac4e4c02d99f15c0efbfe703966fe140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"c67f5535df3dc3502025fcc409699feb2bfc36c4adc644e0b97406a0edaab71d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000700140\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nqkrft7rwghd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nqkrft7rwghd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000700140\u003e: \n    failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nqkrft7rwghd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nqkrft7rwghd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ea1353c268f52134a0f4130e3814846cf91549da4f52d73ca766859449aac2cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"ecae1b0e2c90b770a76b856b5e8af7e32770584b190feb7e81bff82bfbf789f6","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:103]: failed to begin creating HCP cluster \"admin-cred-lifecycle-962qjz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fa1e0\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-962qjz\" in resourcegroup=\"admin-credential-lifecycle-test-7znlrnsb4d2v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-7znlrnsb4d2v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-962qjz\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:103]: failed to begin creating HCP cluster \"admin-cred-lifecycle-962qjz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010fa1e0\u003e: \n    failed starting cluster creation \"admin-cred-lifecycle-962qjz\" in resourcegroup=\"admin-credential-lifecycle-test-7znlrnsb4d2v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-7znlrnsb4d2v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-962qjz\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ec1aeda1db61c6a2b648b44fc23cda37093e57728515e10707b0b55e81e10549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"b1da5cdf26e0eee3ab2b8e8be44d08620bdf20f8433bcd360f87f3dfcf2f9c34","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010120\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-qgdl4c87lvcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-qgdl4c87lvcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010120\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-qgdl4c87lvcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-qgdl4c87lvcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ecfa2dfa7017d5e613b115a584a11e61ddf3576a0ae89e58eed01b8a3df66e06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"16a3295488b90d396bf7321da82f4d92b00ef4b32d39dce5c38e9923b96dbc4c","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d823c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-gf5rj7zc4bks\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gf5rj7zc4bks/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d823c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-gf5rj7zc4bks\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-gf5rj7zc4bks/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"eebe14b7e5b022802eeb5f94c76df593afc9c520769cdcbb0ed8ccc10d77dccb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"e731596e866a4f1aa8f9bf3e441194ce1f7bc0658b366cc2e9ea533c0b061f8a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bv2pbs\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3e1c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-bv2pbs: failed starting cluster creation \"cp-ystream-upgrade-4-21-bv2pbs\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-x9l8w25mj4c7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-x9l8w25mj4c7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-bv2pbs\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bv2pbs\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a3e1c0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-bv2pbs: failed starting cluster creation \"cp-ystream-upgrade-4-21-bv2pbs\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-x9l8w25mj4c7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-x9l8w25mj4c7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-bv2pbs\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fbdf49d87456ecd5f0e0b71ce4ca4e36416365675e267848dfaf0cd3ded36533","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102578870564687872","occurred_at":"2026-09-23T02:00:41Z","failed_at":"e2e","signature_id":"70e94d44bbd3c7bd987ab03782491c93ccc9a29d5f05ba7e1fe547fb16dfc8c7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zf2lwttkqm8b/cluster-candidate-5-0-sfstjx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa2160\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-sfstjx: failed starting cluster creation \"cluster-candidate-5-0-sfstjx\" in resourcegroup=\"rg-candidate-5-0-zf2lwttkqm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-zf2lwttkqm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-sfstjx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-zf2lwttkqm8b/cluster-candidate-5-0-sfstjx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa2160\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-sfstjx: failed starting cluster creation \"cluster-candidate-5-0-sfstjx\" in resourcegroup=\"rg-candidate-5-0-zf2lwttkqm8b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-zf2lwttkqm8b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-sfstjx\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":56,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":56,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}