{"meta":{"date":"2026-09-23","timezone":"UTC","generated_at":"2026-10-31T08:32:20Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":97,"failed_runs":69,"runs_with_occurrences":69,"runs_with_failure_pattern_matches":64,"runs_with_unmatched_occurrences":5,"runs_without_occurrence_rows":28,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"65d675bb300fd87c5f0a14ac91d2d6e076693fe8","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T23:42:18Z","started_at":"2026-09-23T23:42:18Z","completed_at":"2026-09-24T01:52:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:04:21.077156838Z","timing_metadata_checked_at":"2026-09-24T02:04:21.077156838Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:04:21.080905225Z","region_metadata_checked_at":"2026-09-24T02:04:21.080905225Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1ea7807c6150bfdc24425215651cf3c674289f85fa8a1bf852a4b35a4da92a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102906434395049984","occurred_at":"2026-09-23T23:42:18Z","failed_at":"e2e","signature_id":"9d20053e21637cc884e3f5b506bcfcb59a5c2cd22b2f8ee5ff24cb97bfeb11c4","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001038980\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bjdvsjn552jk\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"name\": \"e74a3a8c-8b66-4ef6-aeeb-66083699b0c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T00:25:27.786214979Z\",\n      \"endTime\": \"2026-09-24T00:36:16.168471473Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-bjdvsjn552jk/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-16/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"5ed56c9c71438fc12bb60f11851724f51fb25982","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T23:26:51Z","started_at":"2026-09-23T23:26:51Z","completed_at":"2026-09-24T01:23:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T01:33:52.001171167Z","timing_metadata_checked_at":"2026-09-24T01:33:52.001171167Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T01:33:52.004913354Z","region_metadata_checked_at":"2026-09-24T01:33:52.004913354Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6f068d648acbe97065cddd0536480d056836c090b7aec872bc2337eeba3a1451","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102902545511878656","occurred_at":"2026-09-23T23:26:51Z","failed_at":"e2e","signature_id":"c8ce66f37f9d13dc657060400db223a90301ac4438226c58a0c01cac44d1ec13","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cc06c0\u003e: \n    clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later.\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537","failure_pattern":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later","search_query":"clusterIssuer not ready: reason=ErrUpdateACMEAccount message=Failed to update ACME account:503 urn:ietf:params:acme:error:rateLimited: Service busy; retry later"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1ffc144947159d20ed7dd5010cb0a93b810641281a5d037d76b1920de5d6d537"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:58:15Z","started_at":"2026-09-23T22:58:15Z","completed_at":"2026-09-23T23:32:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:37:40.107438549Z","timing_metadata_checked_at":"2026-09-23T23:37:40.107438549Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:37:40.111155837Z","region_metadata_checked_at":"2026-09-23T23:37:40.111155837Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ef1d48548595682424e0987bbda06126314c2cddc3e79f2dc6774c84dc69724b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102895347519983616","occurred_at":"2026-09-23T22:58:15Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T23:25:15.442Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T23:25:17.399Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T23:25:18.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9983616%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe\ntime=2026-09-23T23:27:49.191Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9983616/providers/Microsoft.Resources/deployments/72a5eef9024c0fb803ba8efe57568ddcaa2d06a84e61744f2eeaeba881052bfe/operationStatuses/08584114009676534307\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T22:12:26Z","started_at":"2026-09-23T22:12:26Z","completed_at":"2026-09-24T00:28:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:40:41.327495438Z","timing_metadata_checked_at":"2026-09-24T00:40:41.327495438Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:40:41.330671228Z","region_metadata_checked_at":"2026-09-24T00:40:41.330671228Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"430f010f6d1f2a44b7240892451e543779c2a61277f763e28c6efce15a5f460f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"alert","signature_id":"125273ad41e8d32bfd551d1beb0bc45baf8569efc2fb05f6f2496e0c967d64ac","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T23:16:20Z\n  Ended:       2026-09-23T23:41:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7507200-svc\", component=\"backend\", name=\"datadump\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"e2cb31fa4f45c3e266ee146be5d7f61e1db49c5f03696eb476670c8afd640f35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"e2e","signature_id":"0c93d7531b51f1ea100516ccd01b0a1c6615ecf8a2bbdc599e9f78e9e71a81e6","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f0c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-94qwsncfglvx\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"name\": \"9a2c4ba6-7b02-45b6-b246-27301c3c7b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:07:02.752849186Z\",\n      \"endTime\": \"2026-09-23T23:26:11.541839688Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7cb5e18e18a5375243672eb6ca5a8a28d188dcd703c199519379851de29ef9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102883816627507200","occurred_at":"2026-09-23T22:12:26Z","failed_at":"alert","signature_id":"45c5b936de9aef65cab348b5a9c37cd387b152250ed997df53a6e8778dabfb08","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T00:08:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7507200-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rpfwd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rpfwd (namespace arobit) on cluster ci01-j7507200-svc has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"45619559b0273524877730f5066212561774a32d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:03:16Z","started_at":"2026-09-23T22:03:16Z","completed_at":"2026-09-24T00:16:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T00:30:24.492408695Z","timing_metadata_checked_at":"2026-09-24T00:30:24.492408695Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T00:30:24.496705081Z","region_metadata_checked_at":"2026-09-24T00:30:24.496705081Z"},"failed_at":["alert","e2e"],"failed_test_count":15,"occurrences":[{"row_id":"076deb2bb8db23bd54067a1e7d49c41108f6e58cb080277d2db57ffcf850f19a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"0c65583f9cf2766f075edef8345e43c644c0d3a9639fd26af02b95e481bc76ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n      \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n      \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-96r455f6cgr8/cluster-candidate-4-21-tc5xmb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ba4aa0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tc5xmb: failed waiting for cluster=\"cluster-candidate-4-21-tc5xmb\" in resourcegroup=\"rg-candidate-4-21-96r455f6cgr8\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2add970-69a8-4470-839c-381250f38990\",\n      \"name\": \"c2add970-69a8-4470-839c-381250f38990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.620493086Z\",\n      \"endTime\": \"2026-09-23T23:25:22.742938899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23751d3251cb875ef3907a1d20bc37a85b9f161ea167d6599a7a41a94aed159d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"266bf773c9ea2839b61a117c9276cf7872810cf8b981f8d2081dee6bd5dac94e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"ef425b9fc641abb8b932709dc29ce158807de6ae50f47a33b3db89e10d6a6b93","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362d00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvfnk745g57p\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\",\n      \"name\": \"277377a5-4429-4698-a365-74f878b05216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:00:00.50151725Z\",\n      \"endTime\": \"2026-09-23T23:29:02.685864461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001362d00\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fvfnk745g57p\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/277377a5-4429-4698-a365-74f878b05216\",\n      \"name\": \"277377a5-4429-4698-a365-74f878b05216\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:00:00.50151725Z\",\n      \"endTime\": \"2026-09-23T23:29:02.685864461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"312344568f7eabc86a986b50d033259549f76ac9ce1669463a8e2a7b7e4cb724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"a08dcc436350e9ea389eb3d5efb0662df729d203d30b15f5920616ae5fc224ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-q7fxr6l2s4gv/cluster-candidate-4-23-8fnk5c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f92780\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-8fnk5c: failed waiting for cluster=\"cluster-candidate-4-23-8fnk5c\" in resourcegroup=\"rg-candidate-4-23-q7fxr6l2s4gv\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"name\": \"f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:55:37.049841827Z\",\n      \"endTime\": \"2026-09-23T23:24:43.011652216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-q7fxr6l2s4gv/cluster-candidate-4-23-8fnk5c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f92780\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-8fnk5c: failed waiting for cluster=\"cluster-candidate-4-23-8fnk5c\" in resourcegroup=\"rg-candidate-4-23-q7fxr6l2s4gv\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"name\": \"f137ea92-b35a-4536-ad97-053b360f7eb9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:55:37.049841827Z\",\n      \"endTime\": \"2026-09-23T23:24:43.011652216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3dbc36fcf4671bf9b06d29711c707f4fe9adba325f6772fd56d1984b41843889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"d8dcd8028ce1adacc03f46fb0158f289f63ae31f72e8e4ce40b790b255cba87a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696ea0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cbz2zvj9pgcc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"name\": \"8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:17.979672195Z\",\n      \"endTime\": \"2026-09-23T23:51:22.771798672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000696ea0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cbz2zvj9pgcc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"name\": \"8aecc921-f63e-4605-92ac-6c0825ca5e73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:17.979672195Z\",\n      \"endTime\": \"2026-09-23T23:51:22.771798672Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"437b11c1003b57bc8c736bd38e421f561b1e74ecf24940ebcf6cbbc75c7092f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"7f9ed0d0bd04a4b470fe9147f2b70a32e6b959bfcc09d740b923b169f280b98f","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:23:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:23:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T23:31:17Z\n  Ended:       2026-09-23T23:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t10muqh402iu8cpp36j8dibe4fqfcp3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T23:32:18Z\n  Ended:       2026-09-23T23:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t10nsc5krmo0ki0tnfsecad8tsn2pgc/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5e36a55b276663958b0731618b72cebe5b692e1584f517184003a78f0448bb5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"0a6194beb78b8c406f21d0c39e8f4938c36409cacbef3176434d6c29c859c82b","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000854de0\u003e: \n    failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n      \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n      \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--8w2v2h7tbr7x/cluster-hs-pre-9brzc9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000854de0\u003e: \n    failed to create HCP cluster cluster-hs-pre-9brzc9: failed waiting for cluster=\"cluster-hs-pre-9brzc9\" in resourcegroup=\"rg-hs-pre--8w2v2h7tbr7x\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6abdd772-bc76-4739-9454-989e477a880a\",\n      \"name\": \"6abdd772-bc76-4739-9454-989e477a880a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.609215386Z\",\n      \"endTime\": \"2026-09-23T23:25:22.953425865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88276530cb10d38731d3e2465e68aab34992770bb41aa55a68b8fd83516b10e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"cb11f1de63b86677e85bb10ffef4875807561668b3123fc194016353354c7ac8","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001006fc0\u003e: \n    failed to create HCP cluster np-dg-minor-4p9zqn: failed waiting for cluster=\"np-dg-minor-4p9zqn\" in resourcegroup=\"rg-np-dg-minor-4p9zqn-9pzvmjjxxgwg\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"name\": \"95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:20:11.073191713Z\",\n      \"endTime\": \"2026-09-23T23:49:12.786080022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001006fc0\u003e: \n    failed to create HCP cluster np-dg-minor-4p9zqn: failed waiting for cluster=\"np-dg-minor-4p9zqn\" in resourcegroup=\"rg-np-dg-minor-4p9zqn-9pzvmjjxxgwg\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"name\": \"95b275e8-f395-482d-b8d4-ca393de42355\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:20:11.073191713Z\",\n      \"endTime\": \"2026-09-23T23:49:12.786080022Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89319e042db26af717577bff4e522fc1770f719760147b3b4aef8d31ff17b598","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"813c3b4a34551005c00d10be781d74a14c7bbb31e37f764d60afab74ae06652e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000464ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8l6vvszb7492\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"name\": \"53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:58:01.487898788Z\",\n      \"endTime\": \"2026-09-23T23:27:02.7845863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000464ae0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8l6vvszb7492\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"name\": \"53f6605f-6b79-4454-8e28-fd239f8916df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:58:01.487898788Z\",\n      \"endTime\": \"2026-09-23T23:27:02.7845863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9bd4609fe496c815f9273292b947da68629a4f405a8604329abb7850cf226c4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"c525979fb2d08507f3d446b04e74f3e5b602e0906716c705201cd81002770ba7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009068a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fx77xd4jlzgc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\",\n      \"name\": \"62671277-0174-4102-8a19-fd81304df39c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:18.488865764Z\",\n      \"endTime\": \"2026-09-23T23:25:22.804222005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009068a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fx77xd4jlzgc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62671277-0174-4102-8a19-fd81304df39c\",\n      \"name\": \"62671277-0174-4102-8a19-fd81304df39c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:18.488865764Z\",\n      \"endTime\": \"2026-09-23T23:25:22.804222005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa806b49bd04b18ac8b36f6cc89c4dfd0b51b1f4bd93031189185d825893e020","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"4342f7d49a31fc62a7cd4188188d768f7c7622da6926b4fcd1d52c2f7741506a","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n      \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bf1200\u003e: \n    failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-dx59g4bq6jrp\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"name\": \"a97c4cb5-238d-43ff-b943-3f685b71b6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:16.987949919Z\",\n      \"endTime\": \"2026-09-23T23:25:22.924061821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ab91de0f007b40686b47998f501a28ba6e277273eeed911c6aa81abb5dac636e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"7fc420454baa557ad64e773af2ec483367dc179f35828ca014109dbc19bd2053","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372520\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ff8h8wnjvp8f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"name\": \"2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.324238915Z\",\n      \"endTime\": \"2026-09-23T23:25:22.950789091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001372520\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ff8h8wnjvp8f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"name\": \"2410d929-9d91-4c03-aaf0-1103e0c4f386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T22:56:17.324238915Z\",\n      \"endTime\": \"2026-09-23T23:25:22.950789091Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d1f15ab51d9208634ab1b7d34df111203f59d19789570a458e7b666a7229f66f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"270a9593b205f1bd6398600a798ec9e0946dfbdebb1375f7feaf55d764c6e286","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:58:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-pbxrh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-pbxrh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"ee3001369776d27b42d98f7738278bc7701ccf7902adeef5f375b5ec9330fa7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"e2e","signature_id":"9eb53b927bd56443c24b7b20ca5bab6adea62beccaa9c3e083e3ba49ff15f1b7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ce480\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xcpdnghpn9qj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"name\": \"4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:18.709940835Z\",\n      \"endTime\": \"2026-09-23T23:51:22.784957829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ce480\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-xcpdnghpn9qj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"name\": \"4981ec5a-b923-4f46-a09c-4542a63d5e8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T23:22:18.709940835Z\",\n      \"endTime\": \"2026-09-23T23:51:22.784957829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3e55e1f30598c07bc5a0984f17970517152e1fcd91e5cbdb72e9ed50b2242f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881520946515968","occurred_at":"2026-09-23T22:03:16Z","failed_at":"alert","signature_id":"fbe2fd343d3a9197d314d9b85e39c50bd319b22385b4b05cd0ef79ae96f47f43","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:33:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6515968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.157:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6515968/providers/microsoft.monitor/accounts/services-j6515968\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-96rhz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T22:01:19Z","started_at":"2026-09-23T22:01:19Z","completed_at":"2026-09-23T22:38:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:43:43.432811327Z","timing_metadata_checked_at":"2026-09-23T22:43:43.432811327Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:43:43.435766917Z","region_metadata_checked_at":"2026-09-23T22:43:43.435766917Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d6385eedc75f05f9b9b7045b22f9fce4034aa5bb3f5d5a31ef5e64ae1c76cc6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102881020008206336","occurred_at":"2026-09-23T22:01:19Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T22:31:07.038Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T22:31:08.814Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T22:31:10.117Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8206336%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1\ntime=2026-09-23T22:34:11.270Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8206336/providers/Microsoft.Resources/deployments/cd29b4bde2b5b7602cb211d98c814688cd52f3aab95b368210e0a8cfb85babb1/operationStatuses/08584114042161978286\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"3eb10bd2f1feb2c6d62a69754d5b84aa95d9f23b","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T21:27:07Z","started_at":"2026-09-23T21:27:07Z","completed_at":"2026-09-23T23:43:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:59:25.843401859Z","timing_metadata_checked_at":"2026-09-23T23:59:25.843401859Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:59:25.84600615Z","region_metadata_checked_at":"2026-09-23T23:59:25.84600615Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0e077d24ef922eefe9593cbfe5ef73d6b62a666d92679680958e6cb67073bbf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102872411731922944","occurred_at":"2026-09-23T21:27:07Z","failed_at":"alert","signature_id":"aef4de3c1d8c7e8e67123da5e644cdc4d14dd0dceea99369d4b941edf923b506","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T22:48:48Z\n  Ended:       2026-09-23T23:00:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-869c9489f8-dbnnd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-869c9489f8-dbnnd (namespace kube-applier) on cluster ci01-j1922944-mgmt-1 has averaged 124.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:16:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1922944-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z6smt\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z6smt (namespace arobit) on cluster ci01-j1922944-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"open","pr_sha":"06b9f9889389f9b6bf8ca6bd99239ae9cb9ee430","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T21:24:41Z","started_at":"2026-09-23T21:24:41Z","completed_at":"2026-09-23T23:43:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:59:25.417151424Z","timing_metadata_checked_at":"2026-09-23T23:59:25.417151424Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:59:25.420744713Z","region_metadata_checked_at":"2026-09-23T23:59:25.420744713Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2c18064313d4aa4d755b3671f909b3d699dcf73ef53aeaefa3d3e31f167ee63a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102871802198888448","occurred_at":"2026-09-23T21:24:41Z","failed_at":"alert","signature_id":"d71eb8d3e94b0946ce9d45150994a79c1da31f720176e1cb83633088dcca59fb","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T23:11:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8888448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m4cdw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m4cdw (namespace arobit) on cluster ci01-j8888448-svc has averaged 124.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T20:49:32Z","started_at":"2026-09-23T20:49:32Z","completed_at":"2026-09-23T21:15:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:20:50.37086223Z","timing_metadata_checked_at":"2026-09-23T21:20:50.37086223Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:20:50.373413821Z","region_metadata_checked_at":"2026-09-23T21:20:50.373413821Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7a602ff1a6bb548cf367efedf993adb662f529022a3c53aab3005d2e8bb376ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862954004221952","occurred_at":"2026-09-23T20:49:32Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T21:07:35.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T21:07:36.837Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T21:07:38.387Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4221952%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee\ntime=2026-09-23T21:10:39.591Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4221952/providers/Microsoft.Resources/deployments/5d60488de7a83700a82e6fe6e3ffcdc28baad600277f95b8571fc3d6f39546ee/operationStatuses/08584114092280138717\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T20:49:31Z","started_at":"2026-09-23T20:49:31Z","completed_at":"2026-09-23T23:01:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:15:56.108448891Z","timing_metadata_checked_at":"2026-09-23T23:15:56.108448891Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:15:56.11224348Z","region_metadata_checked_at":"2026-09-23T23:15:56.11224348Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f883f79f8b55764234ee4ffc0584490b5c4c9ffe51050e4c7362b3111039a202","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102862951168872448","occurred_at":"2026-09-23T20:49:31Z","failed_at":"alert","signature_id":"a051597c46d9f96e8b20b9310e059e4f799849ba3ec132c05941edb67268ea29","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:47:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8872448-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z6lp4\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z6lp4 (namespace arobit) on cluster ci01-j8872448-svc has averaged 125.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102859350493106176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T20:35:13Z","started_at":"2026-09-23T20:35:13Z","completed_at":"2026-09-23T22:54:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:05:25.364997409Z","timing_metadata_checked_at":"2026-09-23T23:05:25.364997409Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:05:25.368402398Z","region_metadata_checked_at":"2026-09-23T23:05:25.368402398Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102859342163218432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T20:35:11Z","started_at":"2026-09-23T20:35:11Z","completed_at":"2026-09-23T23:01:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T23:15:55.971677911Z","timing_metadata_checked_at":"2026-09-23T23:15:55.971677911Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T23:15:55.979653287Z","region_metadata_checked_at":"2026-09-23T23:15:55.979653287Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:18:53Z","started_at":"2026-09-23T20:18:53Z","completed_at":"2026-09-23T22:48:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:54:38.834260316Z","timing_metadata_checked_at":"2026-09-23T22:54:38.834260316Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:54:38.837581695Z","region_metadata_checked_at":"2026-09-23T22:54:38.837581695Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0c46088f1706b0c9cef2514ca376e7b7bd4608af99ac1240c4e740789a121804","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","failed_at":"alert","signature_id":"23e9b2995fc9ded2e9caf55c0eceabc998d92c6176e04363cf44c1eee0ec317d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:52:45Z\n  Ended:       2026-09-23T22:01:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4706560-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4706560-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"bbcb0abfca75bcbb8fe751bdb7264682b1d61b5e44430261fcd4cfc9355f1bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102855242784706560","occurred_at":"2026-09-23T20:18:53Z","failed_at":"alert","signature_id":"254dd39e2f0e4cec1b71ce202f610ab07b9da883c1b746d73e09e4f13421506a","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T22:30:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4706560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rsvxn\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-rsvxn (namespace arobit) on cluster ci01-j4706560-svc has averaged 123.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6733,"pr_state":"closed","pr_sha":"d68f0ce93df0cac968cceb0b5840311d2eb6670f","final_merged_sha":"7df09e82565752899c9669782b8ac3ab38a07b3f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:14:08Z","started_at":"2026-09-23T20:14:08Z","completed_at":"2026-09-23T20:48:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.457979531Z","timing_metadata_checked_at":"2026-09-23T20:59:01.457979531Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.461297921Z","region_metadata_checked_at":"2026-09-23T20:59:01.461297921Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8e576a23eb4f8515fd21dc9df70986d7ac956a95d6c83f40daa42ac4e1f7f4e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102854046258827264","occurred_at":"2026-09-23T20:14:08Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T20:41:08.730Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-23T20:41:10.239Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-23T20:41:11.440Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8827264%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701\ntime=2026-09-23T20:43:42.398Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8827264/providers/Microsoft.Resources/deployments/c45a9cb8ea1f8b2c4d688227f5292bcfda7025c58f2a3979dc36c7e654c8c701/operationStatuses/08584114108148792156\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n  \\\\\\\"message\\\\\\\": \\\\\\\"The zone 'aro-hcp-dev-shard1-slot-01.hcp.osadev.cloud' is not available.\\\\\\\"\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The zone '\u003chcp-api-host\u003e' is not available.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b802ac4158a1088abf1e136667b0ebf668642fc195abb309df01bcfee9866be4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"closed","pr_sha":"1191c30708aa874c2ef1a55f27fe3330c8bcc8bf","final_merged_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:12:38Z","started_at":"2026-09-23T20:12:38Z","completed_at":"2026-09-23T22:20:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:33:38.090698993Z","timing_metadata_checked_at":"2026-09-23T22:33:38.090698993Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:33:38.094258281Z","region_metadata_checked_at":"2026-09-23T22:33:38.094258281Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2d03499c55e8216bb099d24002bf79e79b6d73eccbd7a3146233f0ddd84530d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102853671329992704","occurred_at":"2026-09-23T20:12:38Z","failed_at":"e2e","signature_id":"5278202e490f8d8822dd976e22de925b821240a96d32e8bb0148f9031ab25a7c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084af40\u003e: \n    DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-ldnr6s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00084af40\u003e: \n    DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-k5xxw.apps.aro.v1c9v0y2y6w8p7n.b97m.aro-hcp-dev-shard0-slot-01.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7116/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102852545683656704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7116,"pr_state":"closed","pr_sha":"e3930b456cc9b031d9dc722fd04de3be63742679","final_merged_sha":"e3930b456cc9b031d9dc722fd04de3be63742679","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T20:08:10Z","started_at":"2026-09-23T20:08:10Z","completed_at":"2026-09-23T22:29:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:43:43.646169524Z","timing_metadata_checked_at":"2026-09-23T22:43:43.646169524Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:43:43.650650309Z","region_metadata_checked_at":"2026-09-23T22:43:43.650650309Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7110,"pr_state":"closed","pr_sha":"a3adee787cd6da9312ca13a764ff882bb046be85","final_merged_sha":"ce66d7285ab6bfc8f735b0e7c24632e94158eb25","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T20:01:44Z","started_at":"2026-09-23T20:01:44Z","completed_at":"2026-09-23T21:48:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:02:11.938577107Z","timing_metadata_checked_at":"2026-09-23T22:02:11.938577107Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:02:11.942248894Z","region_metadata_checked_at":"2026-09-23T22:02:11.942248894Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2125c7cacedb8cb4f1d5067e5d374665bb3117595db086980b19e6c1330af98a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"88d7c43f7ba21a8263fb8b4c871ac116be74ea1e4b599156e118374c23dbf6bf","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n      \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004d5460\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8fx9vqqhjrvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"name\": \"5ca16a72-f387-4f26-a7d9-a173356a7d9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:52:19.857894326Z\",\n      \"endTime\": \"2026-09-23T21:11:21.209971005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"44f09923d562e27a236d36bc1632c33c05c0b0520c5b52985a2e7e01298b8152","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"35009a2d8ec15af6629b3b447ac330d1c5376735efbdae49f03535446218b76a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ae83c0\u003e: \n    failed to create customer-infra: failed creating deployment \"customer-infra-cilium-cl-kvs4jp\" in resourcegroup=\"cni-cilium-vbnz9zxk9mkz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: DeploymentActive\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentActive\",\n        \"message\": \"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-vbnz9zxk9mkz/providers/Microsoft.Resources/deployments/customer-infra-cilium-cl-kvs4jp' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/23/2026 8:41:53 PM' with correlationId '5bc6c0ba-196c-4242-b0a0-9410a2c0ee3b', and will expire at '9/30/2026 8:41:53 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive"}},{"row_id":"86c9b946b34abd6efbdf38c85270957a16adbe8a55f2a532bb3416a6ccea5cdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"alert","signature_id":"d5d990830365516d1467804cae51c493981b29d7c2296cdf7adf893366ce4712","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:26:02Z\n  Ended:       2026-09-23T21:34:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4857344-svc\", component=\"frontend\", method=\"post\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4857344-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"ff76d3c516feffc1d05d7f131f77afbdb669a31b0b45c409ea0002b62bfc9e8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850933644857344","occurred_at":"2026-09-23T20:01:44Z","failed_at":"e2e","signature_id":"05cb3f86a6f99432ca7873642f1755e0d1de74704c579e10519fe31b0a3ab239","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc002901d30\u003e: \n    run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","failure_pattern":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790197778134945960\" failed with exit code 6 on VM \"private-ingress-test-vm\" (output: \"000\")"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4e0bd8e13cb37d9a0194e77a0943faa52db03c4ceb721253ee7a21fbde2f97e0","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","c8187cab875ebef3c2972bec23c80766d00e3b845998a754a4922f71837fefdf"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6837/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102850353635528704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6837,"pr_state":"closed","pr_sha":"c130c3ebb8cf261cdc06d58875ff85ef5ce110ea","final_merged_sha":"c130c3ebb8cf261cdc06d58875ff85ef5ce110ea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:59:27Z","started_at":"2026-09-23T19:59:27Z","completed_at":"2026-09-23T22:17:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:23:03.069123273Z","timing_metadata_checked_at":"2026-09-23T22:23:03.069123273Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:23:03.073746258Z","region_metadata_checked_at":"2026-09-23T22:23:03.073746258Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:52:03Z","started_at":"2026-09-23T19:52:03Z","completed_at":"2026-09-23T22:12:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T22:23:03.196964052Z","timing_metadata_checked_at":"2026-09-23T22:23:03.196964052Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T22:23:03.199663743Z","region_metadata_checked_at":"2026-09-23T22:23:03.199663743Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"288da12b2fc1a731c02d9309ef9dd65fd56c9495312d708b48136a6b87f01ffd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"e2e","signature_id":"527306b3dde4b6e298931d98ad45937c8ba364a99fa4d7c4de2caf95207dcd18","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n      \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hv5jpf95qtjp/cluster-candidate-4-21-xnt7bq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4a320\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xnt7bq: failed waiting for cluster=\"cluster-candidate-4-21-xnt7bq\" in resourcegroup=\"rg-candidate-4-21-hv5jpf95qtjp\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"name\": \"f4e39e4f-715f-4e75-a247-c1f9fd0ee98c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:49:22.499172304Z\",\n      \"endTime\": \"2026-09-23T21:18:28.082375104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"87069f61b46c6ece6ad6525cf89c29d0b5ef8af791964258d9c83354123d8c5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"9766aea3f661a15c4b1ef2c8ce948e7447e6a77d814fe12d510f6ddd5782ae00","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:01:37Z\n  Ended:       2026-09-23T21:30:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9340032-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a\", pod=\"router-85d7d5999-qx49z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t0uqkogi5je8dvqpd30ug1p31mmvf4h-a9d6f3s9m6a1f1a/router-85d7d5999-qx49z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e1988dc23cebaccf396951ccb5302059122218cb5fdcdedbe06c4e59cd7b9e0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"9768ce66171b43df472ff44180bc3d8e585fee001b4346d3ab9374574a7b151f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T20:59:04Z\n  Ended:       2026-09-23T21:31:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9340032-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9340032-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e918fac622b4b68875b8c8bef71c125c8cdb9545fbb74a97c14781c61e7e7b4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102848491079340032","occurred_at":"2026-09-23T19:52:03Z","failed_at":"alert","signature_id":"71f4c03d6bb4614602efd1761f8bbf73a703154b3346a44390285f109ca0b681","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:21:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:36:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:36:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"datadump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T21:06:19Z\n  Ended:       2026-09-23T21:21:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9340032-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:42:53Z","started_at":"2026-09-23T19:42:53Z","completed_at":"2026-09-23T20:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:27:19.077217498Z","timing_metadata_checked_at":"2026-09-23T20:27:19.077217498Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:27:19.080003889Z","region_metadata_checked_at":"2026-09-23T20:27:19.080003889Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9ef4b565b50ca9ed86fe30a58ef283acdfb6647b2e61e4ddb687f766d3a1f1e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102846182265982976","occurred_at":"2026-09-23T19:42:53Z","failed_at":"provision","signature_id":"b83201cdb7e435a0abab8639b944505f58098f612295a321d14698e824fe6511","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T20:05:19.475Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-23T20:10:16.316Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-23T20:05:19.482013132Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.259341341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-23T20:05:21.679397764Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314281955Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":\\\"Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/deployments/aks-cluster-create/operations?api-version=2025-04-01\\\\\\\": stream error: stream ID 41; CANCEL; received from peer\\\"}\\n{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620","failure_pattern":"{\"level\":\"ERROR\",\"msg\":\"run failed\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5982976-mgmt-2\",\"clusterName\":\"ci01-j5982976-mgmt-2\",\"region\":\"westus3\",\"err\":\"waiting for deployment","search_query":"{\\\"time\\\":\\\"2026-09-23T20:10:16.314392598Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5982976-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5982976-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: Get \\\\\\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5982976-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584114129632610784?api-version=2025-04-01\u0026t=639257907264738192\u0026c=MIIIJzCCBw-gAwIBAgIRAKCxVbScFDiCR8oT6qcGYJMwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UEAxMrQ0NNRSBHMSBUTFMgUlNBIDIwNDggU0hBMjU2IDIwNDkgV0NVUyBDQSAwMTAeFw0yNjA5MTkwMDU4MzJaFw0yNjEyMTQwNjU4MzJaMEAxPjA8BgNVBAMTNWFzeW5jb3BlcmF0aW9uc2lnbmluZ2NlcnRpZmljYXRlLm1hbmFnZW1lbnQuYXp1cmUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFNlTnLezYuVV9JBgdhqA91jRJUtR4AjUkUKP5Nk7STlDt0_Hwx2OHYvMcuSX2B7klrS1nxF26zm4lnQKcCJTslJYfDrx756nLb5iGS6OS0koE0rze5-sUiu_IHHt28tpGv_9lvJvRWPDZc1uYCzrGyBpz6HvBzVgl3BeyWo9RTZYJhgBVNIATJuDRUzovVt8XczDob4euvsqOVr7rBx6wvKf79ASaeXB6PbT19HGywTrd9Xkd33sWfzQu5ZuHtSgX3i1sp_4BVE8VTuCkqIku5kzbulG3jvlSrsOmn680vsH3sS1APY1L7SymeuyNuVIv20HZmQukmjreOzZu-iGQIDAQABo4IFJDCCBSAwgZ0GA1UdIASBlTCBkjAMBgorBgEEAYI3ewEBMGYGCisGAQQBgjd7AgIwWDBWBggrBgEFBQcCAjBKHkgAMwAzAGUAMAAxADkAMgAxAC0ANABkADYANAAtADQAZgA4AGMALQBhADAANQA1AC0ANQBiAGQAYQBmAGYAZAA1AGUAMwAzAGQwDAYKKwYBBAGCN3sDAjAMBgorBgEEAYI3ewQCMAwGA1UdEwEB_wQCMAAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA4GA1UdDwEB_wQEAwIFoDAdBgNVHQ4EFgQU7SRzDdTHXndOcavyYsWupdx4MecwHwYDVR0jBBgwFoAUFNI34PbWfX7djbq6ZasElCXglh0wggH7BgNVHR8EggHyMIIB7jB7oHmgd4Z1aHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMH2ge6B5hndodHRwOi8vc2Vjb25kYXJ5LWNkbi5wa2kuY29yZS53aW5kb3dzLm5ldC93ZXN0Y2VudHJhbHVzL2NybHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS80MS9jdXJyZW50LmNybDBsoGqgaIZmaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3dlc3RjZW50cmFsdXMvY3Jscy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIGBoH-gfYZ7aHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxLzQxL2N1cnJlbnQuY3JsMIICAAYIKwYBBQUHAQEEggHyMIIB7jB-BggrBgEFBQcwAoZyaHR0cDovL3ByaW1hcnktY2RuLnBraS5jb3JlLndpbmRvd3MubmV0L3dlc3RjZW50cmFsdXMvY2FjZXJ0cy9jY21ld2VzdGNlbnRyYWx1c3BraS9jY21ld2VzdGNlbnRyYWx1c2ljYTAxL2NlcnQuY2VyMIGABggrBgEFBQcwAoZ0aHR0cDovL3NlY29uZGFyeS1jZG4ucGtpLmNvcmUud2luZG93cy5uZXQvd2VzdGNlbnRyYWx1cy9jYWNlcnRzL2NjbWV3ZXN0Y2VudHJhbHVzcGtpL2NjbWV3ZXN0Y2VudHJhbHVzaWNhMDEvY2VydC5jZXIwbwYIKwYBBQUHMAKGY2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS93ZXN0Y2VudHJhbHVzL2NhY2VydHMvY2NtZXdlc3RjZW50cmFsdXNwa2kvY2NtZXdlc3RjZW50cmFsdXNpY2EwMS9jZXJ0LmNlcjB4BggrBgEFBQcwAoZsaHR0cDovL2NjbWV3ZXN0Y2VudHJhbHVzcGtpLndlc3RjZW50cmFsdXMucGtpLmNvcmUud2luZG93cy5uZXQvY2VydGlmaWNhdGVBdXRob3JpdGllcy9jY21ld2VzdGNlbnRyYWx1c2ljYTAxMA0GCSqGSIb3DQEBCwUAA4IBAQA5ody_Abz9ik5_3WVrjOb7e5cyPkeuDRHrpQ3VO0TeurG5LwTtQ-Yu-lE3ow2Qq4IbIIP6kpzWMQ_woGCp96pGdwvp7vLu4SYc-Rp-CKqLpJtXBRQ01RDbqBZWPwrrgrfMjTaopq1rEEHOBBqt2X4fPXlf91DULpbYJikwKVt4rSQPFgvzjtIpRSeHkIIw8oA95SBfgvs-CIisoc1h-1J1s6jmYw8Kqgqc-NoVLrvf4qRHrd8_0YK2nmtl5XnHD8a50gNVozHfx52l7NfwV_tYfwIO82lHrNxrpLkb7gSAAoAkma6s2Tjh23fpSj-E9OFOFMeuCwnaykbAggIZp9hc\u0026s=xAFEUIMkN7HqEcaWnjPt98SPv2n8tDGNfsg9g3ETUj427eutsI2z-oSn-NYDEALQ9cMRKqky-VWvsQV0qp29lgMWk_hw47hsrPBP4zu666M_n4lLgU8QDrRKLDzrIYelxUd035HLMh9F5fWNHrGNulgbDCkeZeqOoc6EAne-535SWhie6nA952GikZdg_fD2pGBdL53D40Ycq2WqzPFAtYN4D-nKCFWKG8QoWd59PYeAW-C_ZWVMdATP9LmSEVVMHqMF13HK7-5WRYDFxs-LPKfY7okL22TjrA2RLKxuv1xWiSUxb0QLisSFa7Bqvbbl6syA1C3_0qKLeHs_XDklIA\u0026h=dLBpH5niUgnkOleLXSR6xT4LrPMGJuQLdmK3gutCEoY\\\\\\\": stream error: stream ID 33; CANCEL; received from peer\\\",\\\"elapsed\\\":\\\"4m56.832387486s\\\"}"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8eaae2745978f98ae080df0fcc5b37bd11dd539de8c53dcf70259bcbe794a620"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840328758235136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:19:37Z","started_at":"2026-09-23T19:19:37Z","completed_at":"2026-09-23T21:45:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:51:43.693870682Z","timing_metadata_checked_at":"2026-09-23T21:51:43.693870682Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:51:43.698837566Z","region_metadata_checked_at":"2026-09-23T21:51:43.698837566Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T19:19:30Z","started_at":"2026-09-23T19:19:30Z","completed_at":"2026-09-23T21:36:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:41:34.401949057Z","timing_metadata_checked_at":"2026-09-23T21:41:34.401949057Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:41:34.404847648Z","region_metadata_checked_at":"2026-09-23T21:41:34.404847648Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"049f8a3afeb95eb2006b04c1470856457c5191c031a088329aade1deaa24270e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840297615527936","occurred_at":"2026-09-23T19:19:30Z","failed_at":"alert","signature_id":"8d0ffdcc532373f83199e900f86943923bf9464de78da96a0bc81c5d10748030","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T21:21:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5527936-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-d7xzw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-d7xzw (namespace arobit) on cluster ci01-j5527936-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T19:19:27Z","started_at":"2026-09-23T19:19:27Z","completed_at":"2026-09-23T21:35:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:41:34.267111899Z","timing_metadata_checked_at":"2026-09-23T21:41:34.267111899Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:41:34.273332779Z","region_metadata_checked_at":"2026-09-23T21:41:34.273332779Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2cda3228746978234f3989339799c220df7461d9ce34fea4f5f7fbc725df6f63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","failed_at":"e2e","signature_id":"3fec0289f4e859b2672b4ecfd1c53de11f801ca25e568cb6fd49601565fabc04","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-ks79lqvdvdz4/cluster-candidate-4-21-tjs922 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b96700\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tjs922: failed waiting for cluster=\"cluster-candidate-4-21-tjs922\" in resourcegroup=\"rg-candidate-4-21-ks79lqvdvdz4\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"name\": \"6f164752-446b-48ce-a79b-2e5f890f02c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T20:08:29.367613866Z\",\n      \"endTime\": \"2026-09-23T20:37:38.881566969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, packageserver, olm-operator, ingress-operator, catalog-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-network-operator, dns-operator, ingress-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e608ced051e13f5963c01177b86dc7083dc3ec405629cacf72c4d113e7a2627c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840286248964096","occurred_at":"2026-09-23T19:19:27Z","failed_at":"alert","signature_id":"f01e8023db70ac4eb379611e2c78c4d54f1aeadca78f25f3826d0ffec54edaa2","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T21:11:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8964096-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dhxkp\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-dhxkp (namespace arobit) on cluster ci01-j8964096-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b6fc32628c7943fbfb2873910bf615def7fec7cbe037b84d079cb918fd32cbe8","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102840280725065728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7114,"pr_state":"closed","pr_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","final_merged_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T19:19:26Z","started_at":"2026-09-23T19:19:26Z","completed_at":"2026-09-23T21:36:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:51:43.844260175Z","timing_metadata_checked_at":"2026-09-23T21:51:43.844260175Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:51:43.847559664Z","region_metadata_checked_at":"2026-09-23T21:51:43.847559664Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102835259199459328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"932144cafaab4d6ac2c6ccac83cf8e59c8e51edb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T18:59:29Z","started_at":"2026-09-23T18:59:29Z","completed_at":"2026-09-23T21:12:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T21:20:50.20722657Z","timing_metadata_checked_at":"2026-09-23T21:20:50.20722657Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T21:20:50.211184557Z","region_metadata_checked_at":"2026-09-23T21:20:50.211184557Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"open","pr_sha":"f0dfa64a174a3416406d82d8241fb7426d3e8153","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T18:35:44Z","started_at":"2026-09-23T18:35:44Z","completed_at":"2026-09-23T20:54:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.592813505Z","timing_metadata_checked_at":"2026-09-23T20:59:01.592813505Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.595513696Z","region_metadata_checked_at":"2026-09-23T20:59:01.595513696Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"59c64f763f8131224bd3746a78983b1dd5e6ac20fc51b52a32f73d52f809c9f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","failed_at":"alert","signature_id":"312cea2b73363d1b160f9cce1d956a42a926ffe6d5a41922d416b5287eec53f4","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T20:25:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4862464-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6s587\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-6s587 (namespace arobit) on cluster ci01-j4862464-svc has averaged 125.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"a8243d83bccfa56e550504ecf298a6d5ea6037918b8a194396659f6167827d27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102829281724862464","occurred_at":"2026-09-23T18:35:44Z","failed_at":"alert","signature_id":"766b1d46f5072f7c906c6126dd0836e5f198da05367c355a94ce2d777ad7547e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T19:49:29Z\n  Ended:       2026-09-23T20:23:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4862464-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4862464-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825931050586112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"open","pr_sha":"1ff09f1b1c644ee60c41a9b6ce0baafd2280068e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T18:22:25Z","started_at":"2026-09-23T18:22:25Z","completed_at":"2026-09-23T20:48:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:59:01.732982061Z","timing_metadata_checked_at":"2026-09-23T20:59:01.732982061Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:59:01.736108951Z","region_metadata_checked_at":"2026-09-23T20:59:01.736108951Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825299610701824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T18:19:54Z","started_at":"2026-09-23T18:19:54Z","completed_at":"2026-09-23T18:48:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:12.010947298Z","timing_metadata_checked_at":"2026-09-23T18:52:12.010947298Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T18:52:12.013930388Z","region_metadata_checked_at":"2026-09-23T19:54:40.427491975Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dc6040c2957dc298543424875eafcc0af051fdf89d7f2d91edd92e7e0c186e60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102825299610701824","occurred_at":"2026-09-23T18:19:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"30be60dc7d1a742de6c8ee95366914897d94f6eb","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T18:09:20Z","started_at":"2026-09-23T18:09:20Z","completed_at":"2026-09-23T20:17:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:27:19.234682177Z","timing_metadata_checked_at":"2026-09-23T20:27:19.234682177Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:27:19.237546368Z","region_metadata_checked_at":"2026-09-23T20:27:19.237546368Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fc2abd0d1661a3c4ae4e0fd028ce0ce09f68f71c2ec1759f04869dc5bb5a2a90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102822640707178496","occurred_at":"2026-09-23T18:09:20Z","failed_at":"e2e","signature_id":"a7690e671288365fae42a54239a1fd3d4683bfffda4c651366b6b6d63b190ce6","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d7cd00\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-c9qhwvtrnfw9\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"name\": \"cce602d9-c8f0-4f8b-9d73-4b32f73a4a7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:51:58.738258798Z\",\n      \"endTime\": \"2026-09-23T19:01:09.161920644Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-c9qhwvtrnfw9/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-20/subnets/customer-vnet-subnet-4-20\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard2-03-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T17:54:24Z","started_at":"2026-09-23T17:54:24Z","completed_at":"2026-09-23T19:55:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:05:32.713251638Z","timing_metadata_checked_at":"2026-09-23T20:05:32.713251638Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:05:32.716568127Z","region_metadata_checked_at":"2026-09-23T20:05:32.716568127Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8cfbcb69b92bcf646b23ae81033bebb5fcd9f63391da41a8edf52e3429f3cab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818882191364096","occurred_at":"2026-09-23T17:54:24Z","failed_at":"e2e","signature_id":"d95e88444841564bfebc7b4d86984dd9a38294a94ca9cf72a10f4adf98342abd","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006be420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n      \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:177]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006be420\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-4wt68pfrx42w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"name\": \"adb5be9c-18bf-41aa-925f-6eb547095403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:46:32.650200261Z\",\n      \"endTime\": \"2026-09-23T19:05:38.929512243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:52:29Z","started_at":"2026-09-23T17:52:29Z","completed_at":"2026-09-23T20:10:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T20:16:28.618192962Z","timing_metadata_checked_at":"2026-09-23T20:16:28.618192962Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T20:16:28.622704146Z","region_metadata_checked_at":"2026-09-23T20:16:28.622704146Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a15f14399161f97da2ef8a309986d00cdc999dc3e5acb7da164323732285c68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102818397388541952","occurred_at":"2026-09-23T17:52:29Z","failed_at":"e2e","signature_id":"b269b2a196f9ad9cd1702b095e9f309573b62770b7cab43403d5312cc9615be1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00121bf40\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:37971-\u003e172.30.0.10:53: read: connection refused\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:49:53Z","started_at":"2026-09-23T17:49:53Z","completed_at":"2026-09-23T18:43:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:12.188337813Z","timing_metadata_checked_at":"2026-09-23T18:52:12.188337813Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:12.190782405Z","region_metadata_checked_at":"2026-09-23T18:52:12.190782405Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"387b14b8dd4f9e51dd930a4fe612d59829a9353b0379e06c85422427cd165d8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102817746126376960","occurred_at":"2026-09-23T17:49:53Z","failed_at":"provision","signature_id":"aabbcc585420cff60362100bdaf09336ff0307d5c296c6855ec0f5fe21f850ce","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/svc-stamped/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T18:34:52.490Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 values=\"map[cloudEnvironmentName:AzurePublicCloud cosmos:map[name:arohcpci01-rp-j6376960 url:https://arohcpci01-rp-j6376960.documents.azure.com:443/] image:map[digest:sha256:6bc5ed31e42afccc2abd6b21b817cc402be829c20f8cc00e71fde01c9be1523a registry:arohcpsvcdev.azurecr.io repository:ci-op-6hlkxni0/pipeline] registration:map[aksResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j6376960-mgmt-1 autoApprove:true hostedClustersManagedIdentitiesKeyVaultURL:https://ah-ci01-mi-j6376960-1.vault.azure.net/ hostedClustersSecretsKeyVaultManagedIdentityClientID:b2cc94cd-3e8b-42a9-bf2a-e025837a95cd hostedClustersSecretsKeyVaultURL:https://ah-ci01-cx-j6376960-1.vault.azure.net/ kubeApplierCosmosContainerName:Manifests-MC-1 maestroConsumerName:hcp-underlay-aro-hcp-dev-shard2-slot-01-mgmt-1 maestroGRPCTarget:maestro-grpc.maestro.svc.cluster.local:8090 maestroRESTAPIURL:http://maestro.maestro.svc.cluster.local:8000 provisionShardStatus:active publicDNSZoneResourceID:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6376960/providers/Microsoft.Network/dnsZones/aro-hcp-dev-shard2-slot-01.hcp.osadev.cloud stampIdentifier:1] serviceAccountName:fleet]\"\ntime=2026-09-23T18:34:52.977Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:34:54.319Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 namespace=fleet\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 dryRun=false\ntime=2026-09-23T18:34:54.894Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1\ntime=2026-09-23T18:35:05.401Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 numVersions=0\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"\\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"\ntime=2026-09-23T18:35:05.530Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=svc-stamped step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38","failure_pattern":"failed to roll out Helm release: \"fleet-reg-ci01-j6376960-mgmt-1\" has no deployed releases","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: \\\"fleet-reg-ci01-j6376960-mgmt-1\\\" has no deployed releases"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["acb0571a19c874c4edfc6efce4cf429757c85c1334f4e11589a3958d721e5d38"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T17:28:39Z","started_at":"2026-09-23T17:28:39Z","completed_at":"2026-09-23T19:43:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:54:40.593347954Z","timing_metadata_checked_at":"2026-09-23T19:54:40.593347954Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:54:40.597498241Z","region_metadata_checked_at":"2026-09-23T19:54:40.597498241Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"5c82024eb87d47d5728ca00a2482dc960b51279e5d62991eafaaeb3b7bb0e73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102812400645312512","occurred_at":"2026-09-23T17:28:39Z","failed_at":"alert","signature_id":"c1db43e1f9ed556a1e717aa9ea521e513132026799cb40e29c7cc1976942dff0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T18:53:46Z\n  Ended:       2026-09-23T19:08:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j5312512-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j5312512-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7117,"pr_state":"closed","pr_sha":"ea43f7ba6df9f089661d7c30a490eb1e026104e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T17:26:00Z","started_at":"2026-09-23T17:26:00Z","completed_at":"2026-09-23T19:41:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:54:40.770229997Z","timing_metadata_checked_at":"2026-09-23T19:54:40.770229997Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:54:40.772861789Z","region_metadata_checked_at":"2026-09-23T19:54:40.772861789Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"27ff72046bb52fb7d0d7fbc701b6fa3eff5233a11846607241a6305c9bf68f8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"e2e","signature_id":"23493215ee1a820f0efaa9de36d3f924d4bf22faf75dcf25d674d9fd4bee714f","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000ca99a0\u003e: \n    Operation cannot be fulfilled on nodes \"t9c6r3e7k2o1w9o-cordon-np-st9vc-2hvv8\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}},{"row_id":"2f62dd92bed9e66582eab23ab554d6abf339c407e81aa301bfcb65cbe160a27a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"alert","signature_id":"cedc29dd595c13bb38b5f8171952695156bf1b2712e54cef49d283d640a3a08e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T19:18:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4456960-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ltf9x\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-ltf9x (namespace arobit) on cluster ci01-j4456960-svc has averaged 125% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"717bd2a6551a73e19b4c02e2e56f8d8bc799bdd4e4439f1455907bab661aabd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"alert","signature_id":"56b2ff341978727c646e219e23fba2dfc1ef2e5ccc4828702e0dece8d2810ba6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T18:28:51Z\n  Ended:       2026-09-23T19:02:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4456960-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4456960-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"8402bcb01e7cb18edcf2352d8e3571f186edf06483a251f462bf537f1c95351d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102811735164456960","occurred_at":"2026-09-23T17:26:00Z","failed_at":"e2e","signature_id":"48a8490fb0e062a5d2a53db5bf3f5a92fd4adef796973aafbfa795c3fe0dbe08","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a21860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n      \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000a21860\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp.aro-hcp-dev-shard1-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"name\": \"52906a0e-e465-4c26-861d-ab3698d37a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T18:28:44.991686145Z\",\n      \"endTime\": \"2026-09-23T18:47:50.145290668Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6925,"pr_state":"open","pr_sha":"9ef5982dac7a7b22811f15404810c31d0372579b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:43:09Z","started_at":"2026-09-23T16:43:09Z","completed_at":"2026-09-23T18:51:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T19:03:01.998237305Z","timing_metadata_checked_at":"2026-09-23T19:03:01.998237305Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T19:03:02.005257782Z","region_metadata_checked_at":"2026-09-23T19:03:02.005257782Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c9d8a130d91ea03de00bcfd1d700efb07ab41edf36fe56ee29600f68dab88c04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800950539325440","occurred_at":"2026-09-23T16:43:09Z","failed_at":"alert","signature_id":"f1b24a151cda52cd0d54ea6a4677f8fb34ef83c5bdadd527eb334c9d72e78bde","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T18:30:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9325440-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nc4vq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-nc4vq (namespace arobit) on cluster ci01-j9325440-svc has averaged 124% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7114,"pr_state":"closed","pr_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","final_merged_sha":"6baaf20b9b4cca8fa1ac355389b9576883164913","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:40:45Z","started_at":"2026-09-23T16:40:45Z","completed_at":"2026-09-23T18:41:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.701287242Z","timing_metadata_checked_at":"2026-09-23T18:52:13.701287242Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.704425432Z","region_metadata_checked_at":"2026-09-23T18:52:13.704425432Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"297538984fb50a279dfb92919b5102e540682b0dd82696d8c2cfa2bf65f43fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7114/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800348405043200","occurred_at":"2026-09-23T16:40:45Z","failed_at":"e2e","signature_id":"8a478233ef9f4819c24a5dc34485444f654a99c79dc829ccb980b0bc2625e398","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b97da0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8g5rnldzw776\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"name\": \"607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:42:05.950445215Z\",\n      \"endTime\": \"2026-09-23T18:01:06.854588111Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b97da0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-8g5rnldzw776\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"name\": \"607fa875-dbd2-4396-9201-8fdd255f0980\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:42:05.950445215Z\",\n      \"endTime\": \"2026-09-23T18:01:06.854588111Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:39:42Z","started_at":"2026-09-23T16:39:42Z","completed_at":"2026-09-23T18:45:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.557195214Z","timing_metadata_checked_at":"2026-09-23T18:52:13.557195214Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.559930305Z","region_metadata_checked_at":"2026-09-23T18:52:13.559930305Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f0d674dbebb4c47bc310791ffaaa65927e17896ff0783e6c30b1c633998e8f0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102800084281331712","occurred_at":"2026-09-23T16:39:42Z","failed_at":"alert","signature_id":"aca35d9e741dfeba834ff114bffafa2a455703b5a32f415b37bdcbd261eb6a67","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T17:51:27Z\n  Ended:       2026-09-23T18:00:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1331712-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1331712-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102799578414714880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"466e2e9244431d18141c0e26224633aa9f8122d0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T16:37:42Z","started_at":"2026-09-23T16:37:42Z","completed_at":"2026-09-23T18:40:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:52:13.42681784Z","timing_metadata_checked_at":"2026-09-23T18:52:13.42681784Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:52:13.429739731Z","region_metadata_checked_at":"2026-09-23T18:52:13.429739731Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795320147578880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T16:20:46Z","started_at":"2026-09-23T16:20:46Z","completed_at":"2026-09-23T18:22:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.349049248Z","timing_metadata_checked_at":"2026-09-23T18:31:59.349049248Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.356817724Z","region_metadata_checked_at":"2026-09-23T18:31:59.356817724Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:20:35Z","started_at":"2026-09-23T16:20:35Z","completed_at":"2026-09-23T18:27:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.528363004Z","timing_metadata_checked_at":"2026-09-23T18:31:59.528363004Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.530941296Z","region_metadata_checked_at":"2026-09-23T18:31:59.530941296Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b6260ae90c2cc05786ae25a4e151ab0ca6388740039250e349fe5f4152a35151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795270910644224","occurred_at":"2026-09-23T16:20:35Z","failed_at":"e2e","signature_id":"2562b6a0aebd3dafa3041bae2bbef4d61a7d09096d6b568a3e7b98b9b1ca7da6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001018240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n      \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001018240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-57j9pfs2vmmr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"name\": \"1d3cf93e-ed99-43b6-aff0-026b8023aa5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T17:11:52.309512854Z\",\n      \"endTime\": \"2026-09-23T17:40:54.426586313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:20:32Z","started_at":"2026-09-23T16:20:32Z","completed_at":"2026-09-23T18:28:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:42:06.771021902Z","timing_metadata_checked_at":"2026-09-23T18:42:06.771021902Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:42:06.773865093Z","region_metadata_checked_at":"2026-09-23T18:42:06.773865093Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"54138909a11d3ecdba2bce14c613918e2342ca0a5592cb34205416ab457a16e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102795261179858944","occurred_at":"2026-09-23T16:20:32Z","failed_at":"alert","signature_id":"c0de1a0ccf1f7966d446e232f88e4d90f26f4017b261cdeb15d15f7eb77aa118","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T17:46:48Z\n  Ended:       2026-09-23T17:57:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9858944-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-6f4c9cc744-8nwkb\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-6f4c9cc744-8nwkb (namespace kube-applier) on cluster ci01-j9858944-mgmt-2 has averaged 126.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102794564984115200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7077,"pr_state":"closed","pr_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","final_merged_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T16:17:46Z","started_at":"2026-09-23T16:17:46Z","completed_at":"2026-09-23T18:21:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:31:59.625986608Z","timing_metadata_checked_at":"2026-09-23T18:31:59.625986608Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:31:59.629440298Z","region_metadata_checked_at":"2026-09-23T18:31:59.629440298Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102792596072960000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T16:09:57Z","started_at":"2026-09-23T16:09:57Z","completed_at":"2026-09-23T16:23:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.825071746Z","timing_metadata_checked_at":"2026-09-23T16:36:58.825071746Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.827780238Z","region_metadata_checked_at":"2026-09-23T16:36:58.827780238Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c8ce51fae56024717e83ccbad328bfaa255c5b2676f6e1f3eb8d15bf1e24b047","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102792596072960000","occurred_at":"2026-09-23T16:09:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T16:00:38Z","started_at":"2026-09-23T16:00:38Z","completed_at":"2026-09-23T17:54:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T18:00:55.400632312Z","timing_metadata_checked_at":"2026-09-23T18:00:55.400632312Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T18:00:55.4041462Z","region_metadata_checked_at":"2026-09-23T18:00:55.4041462Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"772a1aecdfe2ebf92e56874be2be523e71a6adc38da8e579c0a229365db4f93e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102790414846136320","occurred_at":"2026-09-23T16:00:38Z","failed_at":"e2e","signature_id":"2074bff381a20809ab3d0b56df62ec1c9c4cecad07bc3b02a8f9d837a90a1b6a","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:339]: failed to retrieve serial console logs for VM \"s0t7n3b8m8p4t6a-worker-pfp6w-rth7j\"\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a9b5e0\u003e: \n    expected status 200 OK, got 500: {\n        \"error\": {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Internal server error.\"\n        }\n    }\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506","failure_pattern":"ERROR CODE: InternalServerError; context failed to retrieve serial console logs for VM \"\u003cvm\u003e\"","search_query":"\"message\": \"Internal server error.\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b76ddd45eba98c682618262a77ff4b0f94caf1aee78311c14f58f0e94282f506"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"b542c3a7da99e0f09c6b2e4cfc2b3cb9c2bc7980","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T15:31:45Z","started_at":"2026-09-23T15:31:45Z","completed_at":"2026-09-23T17:30:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:40:02.323890462Z","timing_metadata_checked_at":"2026-09-23T17:40:02.323890462Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:40:02.326927052Z","region_metadata_checked_at":"2026-09-23T17:40:02.326927052Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0a3c0d9132c6ae9b63e97c3f8e749422c5e0106c72c7efe095e188b7763ffa32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","failed_at":"e2e","signature_id":"0c29798c8c1659484795e61241cd7388e309cd731fe979947b39b617cfeb82bb","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"bfc85548cb54340c8476dd84cd1decc1b60e8bc751272b29c6f37a41f91f38cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102782984254918656","occurred_at":"2026-09-23T15:31:45Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:33:04Z\n  Ended:       2026-09-23T16:39:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4918656-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"efbaa775509498dd316f673975db4418a30ad62f","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:56:11Z","started_at":"2026-09-23T14:56:11Z","completed_at":"2026-09-23T16:56:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:07:59.287530719Z","timing_metadata_checked_at":"2026-09-23T17:07:59.287530719Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:07:59.293631401Z","region_metadata_checked_at":"2026-09-23T17:07:59.293631401Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"da7a8a0fd146ac84a973b80124dc195a9733bcc594167fcb4c95302d05b117d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102774033853452288","occurred_at":"2026-09-23T14:56:11Z","failed_at":"e2e","signature_id":"3812816f7f5de01152a0f4bc5ce516ed73308e2f6080e009a2d35e56b303c38c","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.006s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"e18ded374e53071eaf6d1ec31c838986e4bc6e44","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:54:53Z","started_at":"2026-09-23T14:54:53Z","completed_at":"2026-09-23T17:02:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:07:59.157356622Z","timing_metadata_checked_at":"2026-09-23T17:07:59.157356622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:07:59.160642612Z","region_metadata_checked_at":"2026-09-23T17:07:59.160642612Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"a0f821b907c42632e10ca7689b83f85ea79052298bc9b9a9f777da7ecfb844fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"e2e","signature_id":"e0cff06d2d3890e9b05a9d35c28b2aff9003a5f93ca966b808d2a0e175a745b6","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018bec90\u003e: \n    GET https://rp.aro-hcp-dev-shard3-slot-02.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"name\": \"54444a02-0c9c-4321-935a-b1997c03724e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:02.945988001Z\",\n      \"endTime\": \"2026-09-23T16:24:05.016015435Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad4c3242785602103ddf9d0aa115d02dae9c08f9551055e43c550435cf51369b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"alert","signature_id":"d57782e3eb6f6cd2059f92437161a764f82ac19056eed7d813da72181ad2f92d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:15:00Z\n  Ended:       2026-09-23T16:46:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T16:15:00Z\n  Ended:       2026-09-23T16:45:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-jcrjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"c5c269a24369c046230bfcdb2787ff2e337a13aeb3a8ba23daf31c87ba78b9a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"alert","signature_id":"d679a18a9e7a13bb82f0cbe4fdbabac5624367c05fd9b68e2584227db8962e2f","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T16:17:56Z\n  Ended:       2026-09-23T16:44:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n  Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T16:17:56Z\n  Ended:       2026-09-23T16:45:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0621440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.95:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0621440/providers/microsoft.monitor/accounts/hcps-j0621440\", namespace=\"ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0ac0532-43d4-43f8-be90-7de951bc2ef5\"\n  Description: Pod ocm-arohcpci01-2t0qglpi4c0a10c27inccv2cpdf94jj4-e6q8x7z0j2j9n3s/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"d8e65efa8b4362aa79603b17c2892fea71ff8cdff9983290761af55222c5e2f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773722690621440","occurred_at":"2026-09-23T14:54:53Z","failed_at":"e2e","signature_id":"19b631f7d5f2a850bd1bdac6e41017fbcf045a75d8474e6adf13f1324cd43c53","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:155]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013ee480\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-bq2n97vrj5tz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-bq2n97vrj5tz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7087/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102773191775621120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7087,"pr_state":"closed","pr_sha":"efdd1a00e3bd03f1bb647856387c92cf040a29fe","final_merged_sha":"6ef108e37293c8a1a05b1ee1679fda670a90bb09","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T14:52:51Z","started_at":"2026-09-23T14:52:51Z","completed_at":"2026-09-23T17:05:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:18:34.497010397Z","timing_metadata_checked_at":"2026-09-23T17:18:34.497010397Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:18:34.500430386Z","region_metadata_checked_at":"2026-09-23T17:18:34.500430386Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T14:50:10Z","started_at":"2026-09-23T14:50:10Z","completed_at":"2026-09-23T17:09:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T17:18:34.658262464Z","timing_metadata_checked_at":"2026-09-23T17:18:34.658262464Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T17:18:34.660786255Z","region_metadata_checked_at":"2026-09-23T17:18:34.660786255Z"},"failed_at":["alert","e2e"],"failed_test_count":26,"occurrences":[{"row_id":"03aab9bc365cc534c7752ced0f5debd4812f44e341f4220f097043b0cc3c9e83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"fb3c31a801804fd94e8c51eb7bcff2f2193b26a3f6b7ab9373ac03f4a1eac932","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--q72fthqnzbpj/cluster-hs-pre-hbzxm7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b10260\u003e: \n    failed to create HCP cluster cluster-hs-pre-hbzxm7: failed waiting for cluster=\"cluster-hs-pre-hbzxm7\" in resourcegroup=\"rg-hs-pre--q72fthqnzbpj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"name\": \"e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.68921584Z\",\n      \"endTime\": \"2026-09-23T16:22:46.778697738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--q72fthqnzbpj/cluster-hs-pre-hbzxm7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b10260\u003e: \n    failed to create HCP cluster cluster-hs-pre-hbzxm7: failed waiting for cluster=\"cluster-hs-pre-hbzxm7\" in resourcegroup=\"rg-hs-pre--q72fthqnzbpj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"name\": \"e32709e9-bd3d-4b8f-99d7-ef1867b3765b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.68921584Z\",\n      \"endTime\": \"2026-09-23T16:22:46.778697738Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"04a2bba35bdf5f31e066c03c20697bc9ec614b3ee4092efd3acce7ace7b16334","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"2df52c1cafa5532c454b36cb7b084b68593cebfadd2864d45dd1a118200b8d52","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jmdbzj\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0980\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jmdbzj: failed waiting for cluster=\"cluster-zstream-4-21-jmdbzj\" in resourcegroup=\"rg-zstream-upgrade-4-21-4gvzx8nqmkwd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"name\": \"6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.347085548Z\",\n      \"endTime\": \"2026-09-23T16:22:56.388404215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jmdbzj\" with version 4.21.33 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0980\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jmdbzj: failed waiting for cluster=\"cluster-zstream-4-21-jmdbzj\" in resourcegroup=\"rg-zstream-upgrade-4-21-4gvzx8nqmkwd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"name\": \"6fcddd5b-c1a1-4e97-9499-1f7660ea745f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.347085548Z\",\n      \"endTime\": \"2026-09-23T16:22:56.388404215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"08ac124f535a759e298dd9594a0b329f0c12e627046279e746f2aa09b78480ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"449030cbb33da5d4c8dac990dceda44dd62ccc5ca1ffc1ccd12391d7b55e0681","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00055a3c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z85zj8vh8z7t\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"name\": \"fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:38.637505727Z\",\n      \"endTime\": \"2026-09-23T16:23:46.403891132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00055a3c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z85zj8vh8z7t\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"name\": \"fe9c27f3-12ab-47fb-ab19-f97558746925\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:38.637505727Z\",\n      \"endTime\": \"2026-09-23T16:23:46.403891132Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0dfb0368223a888f99738c68a538ce5d96639f292fab5e7abb9471b3f23bb771","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"b4f75fac021e402acc2db264f349c96a35364d535bee0b2e0065ea599920a92f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-t2gnbd\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe480\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-t2gnbd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-t2gnbd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-cbnzwcvqnmsl\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"name\": \"5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.790630075Z\",\n      \"endTime\": \"2026-09-23T16:22:56.599612998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-t2gnbd\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bfe480\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-t2gnbd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-t2gnbd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-cbnzwcvqnmsl\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"name\": \"5e27dfe4-3862-4b07-8ae4-c4377dbf948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.790630075Z\",\n      \"endTime\": \"2026-09-23T16:22:56.599612998Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"251be9ebc0d377de84fb5bd489fb8ae8845df64822d2479dc40a0ae5d2b8c162","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"0adb4bb4ae9202e459b2315ccfce11f495585f3d7bac3fc7a2487d9796add3c7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005383e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lmfj4bczvz8s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"name\": \"13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.078676518Z\",\n      \"endTime\": \"2026-09-23T16:22:46.814369908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005383e0\u003e: \n    failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lmfj4bczvz8s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"name\": \"13b9d58f-8270-477f-aa5e-50f2c6683b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.078676518Z\",\n      \"endTime\": \"2026-09-23T16:22:46.814369908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"294bc2922170c7a1b61a069bae0a2886966661cc46b8b6ffbbb25543ef46b307","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"9e823b348b35d763f4f7278b55c97380508738106688d0cd3c61c152c24157ec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-hj788x\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f801c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-hj788x: failed waiting for cluster=\"cluster-zstream-4-22-hj788x\" in resourcegroup=\"rg-zstream-upgrade-4-22-jdz9qkrk7g87\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"name\": \"cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.267758855Z\",\n      \"endTime\": \"2026-09-23T16:22:46.837301028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-hj788x\" with version 4.22.14 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f801c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-hj788x: failed waiting for cluster=\"cluster-zstream-4-22-hj788x\" in resourcegroup=\"rg-zstream-upgrade-4-22-jdz9qkrk7g87\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"name\": \"cb6ee9cf-b4cb-447c-b859-9eb837187023\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.267758855Z\",\n      \"endTime\": \"2026-09-23T16:22:46.837301028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d074f1de4e240b2a88bfafd50367f03c7eafd6672c012afe30affba9653d556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"e1d6b4b1a86d1907639ae72cc2ccc8137426d24a60ad5b4f6f3e7247ee41a9cc","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbc500\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb9nfmvpzmj4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\",\n      \"name\": \"0f138583-3da6-4491-9246-0255d290efc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.301957583Z\",\n      \"endTime\": \"2026-09-23T16:22:56.376646542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbc500\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kb9nfmvpzmj4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0f138583-3da6-4491-9246-0255d290efc1\",\n      \"name\": \"0f138583-3da6-4491-9246-0255d290efc1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.301957583Z\",\n      \"endTime\": \"2026-09-23T16:22:56.376646542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"342d5e0b4772c14f0ce24853aa1b44bc78d4a1dff23fe5440773775a6c0fbfad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"36a05e474622bc3fcc801d228958fffc4a6d075602e55266d09e5b2e91c19368","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-8w2jrzrjz9pw/cluster-candidate-4-20-wj5jwq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001648460\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wj5jwq: failed waiting for cluster=\"cluster-candidate-4-20-wj5jwq\" in resourcegroup=\"rg-candidate-4-20-8w2jrzrjz9pw\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"name\": \"52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:34.986564037Z\",\n      \"endTime\": \"2026-09-23T16:22:36.376773532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-8w2jrzrjz9pw/cluster-candidate-4-20-wj5jwq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001648460\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wj5jwq: failed waiting for cluster=\"cluster-candidate-4-20-wj5jwq\" in resourcegroup=\"rg-candidate-4-20-8w2jrzrjz9pw\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"name\": \"52fba8b4-23a6-4130-89f0-5fc7e90a72f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:34.986564037Z\",\n      \"endTime\": \"2026-09-23T16:22:36.376773532Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3dd9bd246d279d2117f51e531b7a9812653113a1b3036d50424ecf374be7558c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"97bb982d1e6ea1aa19045a58c60ec413fe3ba3d8834e3141a02e5782483f6785","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f646a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cpzrksbzwbnx\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"name\": \"3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:20.371775923Z\",\n      \"endTime\": \"2026-09-23T16:24:26.21161142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f646a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cpzrksbzwbnx\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"name\": \"3e9be050-bf9c-42a3-967b-1db23fa7c39a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:55:20.371775923Z\",\n      \"endTime\": \"2026-09-23T16:24:26.21161142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f135409be34decbf7cf1ea979cd95f609cc68095d2a1f1685eab4b366ef8b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"2f81637f8265c258ea75aa24f9fd9432ef36114b9ebee46504be7d5770c435b7","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012448d0\u003e: \n    GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"name\": \"c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:30.756948595Z\",\n      \"endTime\": \"2026-09-23T16:23:36.211198776Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012448d0\u003e: \n    GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"name\": \"c4f0afe4-8459-49e1-889d-631d8ffdb648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:54:30.756948595Z\",\n      \"endTime\": \"2026-09-23T16:23:36.211198776Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f79d82e16020a0c88a3e57a3fd3aec0b4911c2cbe6d6e9854a2e2199d232b31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"5f3ee90b05be0d7f1487912ea92f8d3f0441c3abfa713a796fa8f25021c270ac","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24340\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-pp5dtlw88rfs\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"name\": \"bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T16:17:57.447240329Z\",\n      \"endTime\": \"2026-09-23T16:47:00.702969424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24340\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-pp5dtlw88rfs\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"name\": \"bf447353-9164-44e1-b7bb-b41146af7064\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T16:17:57.447240329Z\",\n      \"endTime\": \"2026-09-23T16:47:00.702969424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c0071eb8b13c2a5162e89bb1eb353a066789900906c50095ffabf420f916329","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"a465030c91e7749c08c2c7314b00f14fac042b380238eb18631f6fc0cc7ad978","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107a1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-z5kd6wr2c86f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"name\": \"5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.576887124Z\",\n      \"endTime\": \"2026-09-23T16:22:56.469457208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00107a1e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-z5kd6wr2c86f\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"name\": \"5506d31e-96cb-4248-a2bc-e5375063105d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.576887124Z\",\n      \"endTime\": \"2026-09-23T16:22:56.469457208Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b51e268b6b1f576bbce735ca73aa0d9c4bceb44f5f967934b727dd9a08a783f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"dd70726fbf025a2e22838d8e9a9bcfaa30032d7ec07810dbb606326dc89fe874","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5cc20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8j8ndzvzp2dj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\",\n      \"name\": \"a2be5479-1530-4944-82f9-a854036654b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.658130554Z\",\n      \"endTime\": \"2026-09-23T16:22:56.230648382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5cc20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8j8ndzvzp2dj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a2be5479-1530-4944-82f9-a854036654b2\",\n      \"name\": \"a2be5479-1530-4944-82f9-a854036654b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.658130554Z\",\n      \"endTime\": \"2026-09-23T16:22:56.230648382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"67c53202595f7fc78fd7ba40f496859d9ba76c974210d862db260db22fa46ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"51f8c6265949434e6442ba80f7ade4730dcad12175f31515ce73c25444d377e5","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skmlmr with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00166a640\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skmlmr: failed waiting for cluster=\"np-version-upgrade-cluster-skmlmr\" in resourcegroup=\"rg-np-version-upgrade-skmlmr-gth9wdbm2lgd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"name\": \"edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.126846914Z\",\n      \"endTime\": \"2026-09-23T16:22:56.455577909Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-skmlmr with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00166a640\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-skmlmr: failed waiting for cluster=\"np-version-upgrade-cluster-skmlmr\" in resourcegroup=\"rg-np-version-upgrade-skmlmr-gth9wdbm2lgd\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"name\": \"edca57d8-1d8f-46a8-8263-c347e8c7a645\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.126846914Z\",\n      \"endTime\": \"2026-09-23T16:22:56.455577909Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7185d8e7a8c0d094cc8529d8f0110487422e4c897c82ce5d3536e03cb8a3137f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"d6cea3ffddfab34ef89656bb4911dcfd537e070bc053f39a0b804e5db90f21f8","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012263c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-9v628njd5r4h\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"name\": \"eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.077341251Z\",\n      \"endTime\": \"2026-09-23T16:22:46.744916423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012263c0\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-9v628njd5r4h\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"name\": \"eab71cd7-f7c6-4e17-a391-0a416ce9e57c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:45.077341251Z\",\n      \"endTime\": \"2026-09-23T16:22:46.744916423Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7bdcaacc0c17e5f2a6281ddd22c53324c5309cafe2b3c4a4995b9af4948a9b54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"89edcec5785330a2ee63b672917ff746b2f687ab7ded705d0cc185c95d8bec14","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j694t442mj2r\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"name\": \"f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.255245987Z\",\n      \"endTime\": \"2026-09-23T16:22:56.363291103Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f18e20\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j694t442mj2r\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"name\": \"f6b220ce-3096-4a36-8499-22cf011cf0a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:53.255245987Z\",\n      \"endTime\": \"2026-09-23T16:22:56.363291103Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82732f43ba874be54d8912d3278daebd2f13398829cde0bd596f126c4b26fa3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"9731f91656941bbb9380fa37cca11d45aa6ad2c56b1236e28cb4401092e4c0e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gt4scq at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018268c0\u003e: \n    failed to create HCP cluster np-skip-minor-gt4scq: failed waiting for cluster=\"np-skip-minor-gt4scq\" in resourcegroup=\"rg-np-skip-minor-gt4scq-f9nsllw59lzt\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\",\n      \"name\": \"949773e8-36e8-45db-a78e-a47066415d62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.566834881Z\",\n      \"endTime\": \"2026-09-23T16:22:56.625731333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gt4scq at version 4.22.15\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018268c0\u003e: \n    failed to create HCP cluster np-skip-minor-gt4scq: failed waiting for cluster=\"np-skip-minor-gt4scq\" in resourcegroup=\"rg-np-skip-minor-gt4scq-f9nsllw59lzt\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/949773e8-36e8-45db-a78e-a47066415d62\",\n      \"name\": \"949773e8-36e8-45db-a78e-a47066415d62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.566834881Z\",\n      \"endTime\": \"2026-09-23T16:22:56.625731333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f4a3aa0be444344d70bf52d8eb245aae2b3a86dfd63d4d9d249afe7b78a6112","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"alert","signature_id":"e210a4373a8a1dff8335c12bcbfb22a4bca9ffd1f73200c066c63547ac34373a","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:16:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:16:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2he3nq1jn6qmjkr3aaspbgstbat/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2h9j7mhi4jsf9pvdiin21umprdi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg3radjabcqnahr40ucc01lfmk5a8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qge4i1v30v9fissqi7v4cphc1oevm/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2uevkmvfsffrb6l8hl332p8ardq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg504tgg42v8ako2cd4g5d2nos1ka/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg55h7r4jqvn6q2ve0nvli480gqgj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg4fmbf9m1ge8flue3mi2da0mjinv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg4ehbc8a4kaevorrib9n0fhaggup/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qge4qjfei7usbdki96o5frqdag2t4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-23T16:27:22Z\n  Ended:       2026-09-23T16:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t0qg2pu5vkhroo89ii97sbdd2h2evli/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"922d61aa5578a722be8242cb496c0e263e1b77a2933ad39ee3f4592103e77f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"alert","signature_id":"0b112a92c68becda58d472a3dbc17b517a7589cf579446aec092f675af664746","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:26:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:27:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3938304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.85:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3938304/providers/microsoft.monitor/accounts/services-j3938304\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-k26xd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"a7ba0c727b565e597e877e0d9934aa33a08b0d4c82df6eb91de1fa3414c8aab0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"cfc810c40e60a646985d0ea21bde9b0fc719a1179e342cbce3fe6eb38a62ed69","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7gd6nm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f4660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7gd6nm: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7gd6nm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rrnk84sgmlwn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\",\n      \"name\": \"77c395bf-c417-4455-9807-799065c6b12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.926344123Z\",\n      \"endTime\": \"2026-09-23T16:22:56.209934575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-7gd6nm\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f4660\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-7gd6nm: failed waiting for cluster=\"cp-ystream-upgrade-5-0-7gd6nm\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rrnk84sgmlwn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/77c395bf-c417-4455-9807-799065c6b12d\",\n      \"name\": \"77c395bf-c417-4455-9807-799065c6b12d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.926344123Z\",\n      \"endTime\": \"2026-09-23T16:22:56.209934575Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7f5b5cd255469615d7fdc4c94cd0e36d30cd6fd18157b18d7cc43e208f068b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"6fc513af79b2809dcb5d5df1bdafeac7cf5d4c3a1862412ed3eba47ccae158c5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001164480\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-sg68hlrsd9c4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"name\": \"c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.818708285Z\",\n      \"endTime\": \"2026-09-23T16:22:46.588447682Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001164480\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-sg68hlrsd9c4\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"name\": \"c1a43583-7e84-4614-815f-0f428db0df68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.818708285Z\",\n      \"endTime\": \"2026-09-23T16:22:46.588447682Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c2d97b03a7e81dee96c9c83141626dd2ccb4c44948fc9e7f0ed27e8fe0a7f004","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"1b19c6dab2da849dbdc19f25d8a7c8a64dadf6697567bfe5d10096294dd0a6e4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4240\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tj58cr2mq6ls\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"name\": \"eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.408421024Z\",\n      \"endTime\": \"2026-09-23T16:22:46.864374297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d4240\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-tj58cr2mq6ls\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"name\": \"eb238e5a-3a60-4002-b053-e96c3bdeb2ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:46.408421024Z\",\n      \"endTime\": \"2026-09-23T16:22:46.864374297Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"da7bbf8d0c6b951ab9d14abb2bd01d2eef11fa33ff3532e0fa596c88d9710e76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"381000ecc9d6086c37b0219505b21d93a9c939a2ccf23f03576b38ab84b5a55e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d141e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9lh7k5m447kj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"name\": \"02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.505483407Z\",\n      \"endTime\": \"2026-09-23T16:22:56.544110064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d141e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-9lh7k5m447kj\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"name\": \"02a3231c-7cad-4feb-a266-f06b18888bfe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:50.505483407Z\",\n      \"endTime\": \"2026-09-23T16:22:56.544110064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"db593533e7296bc60d8b9c297d7d75897dcdaaade801ff1656e4d37de1021d90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"cb7663c2933e4c98c3163332183af2e5279eff6073b28c84e67ab60d8693fc1a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-5vk9ph\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017a2760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-5vk9ph: failed waiting for cluster=\"cp-ystream-upgrade-4-22-5vk9ph\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-dw4krrwmbz6s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"name\": \"c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.801712787Z\",\n      \"endTime\": \"2026-09-23T16:22:56.470110002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-5vk9ph\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017a2760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-5vk9ph: failed waiting for cluster=\"cp-ystream-upgrade-4-22-5vk9ph\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-dw4krrwmbz6s\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"name\": \"c6e1dafd-03ab-46f2-99d1-424bbb7b0f7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:52.801712787Z\",\n      \"endTime\": \"2026-09-23T16:22:56.470110002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd2a5fdb09400a11ae9c34643ca8e1c913342005c5c0611d1f1f154e5ce1a897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"e94f7132ddfa2da930718c7e555576608ef511972be7b068ecd1cbc93745767d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102772525283938304","occurred_at":"2026-09-23T14:50:10Z","failed_at":"e2e","signature_id":"47903548f1f143d7ab9998fc3c8c1fbc469323e86e2d9a0d0fddfcc0a731af5d","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f98140\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l95n8mx296wc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"name\": \"7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.243780388Z\",\n      \"endTime\": \"2026-09-23T16:22:56.457631227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f98140\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l95n8mx296wc\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"name\": \"7e3202c3-297e-4668-8e78-60c3e561dd70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T15:53:47.243780388Z\",\n      \"endTime\": \"2026-09-23T16:22:56.457631227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":26,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":26,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:36:53Z","started_at":"2026-09-23T14:36:53Z","completed_at":"2026-09-23T16:33:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:47:21.245116319Z","timing_metadata_checked_at":"2026-09-23T16:47:21.245116319Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:47:21.249519704Z","region_metadata_checked_at":"2026-09-23T16:47:21.249519704Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6a99f29f78230bf3d45bba48fa2f5456a75e1fe745b14b6136b119cfa5954394","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102769175150727168","occurred_at":"2026-09-23T14:36:53Z","failed_at":"alert","signature_id":"3135ce76bbdd1ef05af982db7996ed6329db94eadabfb94067b89a304dc4d2cb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T15:53:20Z\n  Ended:       2026-09-23T15:59:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0727168-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0727168-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102765375618813952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T14:21:41Z","started_at":"2026-09-23T14:21:41Z","completed_at":"2026-09-23T16:23:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.688836874Z","timing_metadata_checked_at":"2026-09-23T16:36:58.688836874Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.693511359Z","region_metadata_checked_at":"2026-09-23T16:36:58.693511359Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"1769127263098290ba69f6f4e9bc0696f1c66e33","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:18:47Z","started_at":"2026-09-23T14:18:47Z","completed_at":"2026-09-23T16:25:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:36:58.525309487Z","timing_metadata_checked_at":"2026-09-23T16:36:58.525309487Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:36:58.529756973Z","region_metadata_checked_at":"2026-09-23T16:36:58.529756973Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"20161848cb282b3d3cba8f3cb394772ad2a3d1f7b2ce18d2cb0e5a31e2806816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102764619155116032","occurred_at":"2026-09-23T14:18:47Z","failed_at":"alert","signature_id":"df2c8e2eea124fae55058fbdd964feff88c2979d18d038ec6b6702d5cd247db8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T16:11:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5116032-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cmbd9\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-cmbd9 (namespace arobit) on cluster ci01-j5116032-svc has averaged 125.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T14:11:30Z","started_at":"2026-09-23T14:11:30Z","completed_at":"2026-09-23T16:08:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:16:10.214826775Z","timing_metadata_checked_at":"2026-09-23T16:16:10.214826775Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:16:10.222905748Z","region_metadata_checked_at":"2026-09-23T16:16:10.222905748Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"46528e9037349e5cf8dc5a037258af3070882d93ced3d15bfac53af4ca0b7bc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102762797665357824","occurred_at":"2026-09-23T14:11:30Z","failed_at":"e2e","signature_id":"8c22befe458ceb42a7ccbeae384be919b61714799a47b6e0ea785ee3e8137ae4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118c460\u003e: \n    DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pt8mvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118c460\u003e: \n    DNS for route host agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv7vn.apps.aro.b3o6y9u3m3w9a8r.7oej.aro-hcp-dev-shard0-slot-00.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-7fjnt.apps.aro.v20260901.jogf.aro-hcp-dev-shard1-slot-04.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:56:43Z","started_at":"2026-09-23T13:56:43Z","completed_at":"2026-09-23T16:04:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:16:10.515642378Z","timing_metadata_checked_at":"2026-09-23T16:16:10.515642378Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:16:10.521513258Z","region_metadata_checked_at":"2026-09-23T16:16:10.521513258Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7f32a2d651aa9c59386e454937a009e1b3b1c29a5395e0236fb4054d5b49e0f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102759065523326976","occurred_at":"2026-09-23T13:56:43Z","failed_at":"alert","signature_id":"b522a7cea6c2812e29c1ea3b0132f66a43bf87236c4f0123b2bd527a73ab14bb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T14:55:12Z\n  Ended:       2026-09-23T15:28:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3326976-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3326976-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102754197291143168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"4ac6a43f989e509deee6e8730933499503439b9a","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:37:22Z","started_at":"2026-09-23T13:37:22Z","completed_at":"2026-09-23T14:28:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:13:19.788575948Z","timing_metadata_checked_at":"2026-09-23T15:13:19.788575948Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:13:19.792359636Z","region_metadata_checked_at":"2026-09-23T15:13:19.792359636Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9c06f43823dd9c82f07acdd590da87ade7f804136d892954c0b278422e2bb16e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102754197291143168","occurred_at":"2026-09-23T13:37:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"11fda0e55e54db52516402aac997f9791cea0a7e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T13:34:43Z","started_at":"2026-09-23T13:34:43Z","completed_at":"2026-09-23T14:23:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:31:08.904547879Z","timing_metadata_checked_at":"2026-09-23T14:31:08.904547879Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:31:08.908040767Z","region_metadata_checked_at":"2026-09-23T14:31:08.908040767Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b581ef52a3a90c861844caae9cf6ca2f3bef9efb41e403e75021ea63a666da3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102753529973182464","occurred_at":"2026-09-23T13:34:43Z","failed_at":"provision","signature_id":"15f55de3f2ea759d42d39cc08a545eae0c5c976acedd44ef9735b3b78e99695d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T14:16:23.495Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-23T14:17:09.991Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: not found\\nCommand failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb","failure_pattern":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp quay.io/app-sre/aro-hcp-clusters-service@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/app-sre/aro-hcp-clusters-service:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json --to-registry-config /tmp/tmp.DLBEJUu7Ue/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["35d3eb882618c30032128a06359768c21f803eaffc0a5e443a0fbb171cbafbdb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102752015808466944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"a9f744ec99dd74bed140edab47cb30ee67d3912e","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T13:28:42Z","started_at":"2026-09-23T13:28:42Z","completed_at":"2026-09-23T15:29:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:44:33.291890491Z","timing_metadata_checked_at":"2026-09-23T15:44:33.291890491Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:44:33.29515818Z","region_metadata_checked_at":"2026-09-23T15:44:33.29515818Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7044,"pr_state":"closed","pr_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","final_merged_sha":"0f625957a86b5fda4999ce65087e506bc2499c89","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:26:28Z","started_at":"2026-09-23T13:26:28Z","completed_at":"2026-09-23T15:50:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T16:05:43.007493418Z","timing_metadata_checked_at":"2026-09-23T16:05:43.007493418Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T16:05:43.011317406Z","region_metadata_checked_at":"2026-09-23T16:05:43.011317406Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"45b7e31d2e54d9afb213bf465c0e350a2adc2dde0b5c558156e356b62eada9d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7044/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751455361372160","occurred_at":"2026-09-23T13:26:28Z","failed_at":"alert","signature_id":"27fba5395347858a8e8e715d32f7f0fe022ff7065fe1ea71ae9a42e0ed9a0a0d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T15:27:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1372160-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-kr2cs\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-kr2cs (namespace arobit) on cluster ci01-j1372160-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7106,"pr_state":"closed","pr_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","final_merged_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:26:24Z","started_at":"2026-09-23T13:26:24Z","completed_at":"2026-09-23T15:32:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:44:33.450260276Z","timing_metadata_checked_at":"2026-09-23T15:44:33.450260276Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:44:33.453626065Z","region_metadata_checked_at":"2026-09-23T15:44:33.453626065Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d2d373686094856f7bdfad8d8d08fdef61c112b681b3c44a34d791d8d9e069b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751439267827712","occurred_at":"2026-09-23T13:26:24Z","failed_at":"e2e","signature_id":"e4012e513d3cafa3f6c8980450c912f0d288e7cf2e0c893549e7a66042041837","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2ede0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2jl588bhfvmd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"name\": \"8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:47:29.897134227Z\",\n      \"endTime\": \"2026-09-23T15:06:35.791795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e2ede0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2jl588bhfvmd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"name\": \"8d975644-73c1-4964-98e7-311fe1b5f246\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:47:29.897134227Z\",\n      \"endTime\": \"2026-09-23T15:06:35.791795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102751270510006272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5493,"pr_state":"open","pr_sha":"10f8327910e5d5309d012a4c304ebcb9f2b4aa74","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T13:25:44Z","started_at":"2026-09-23T13:25:44Z","completed_at":"2026-09-23T15:26:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:33:49.759341373Z","timing_metadata_checked_at":"2026-09-23T15:33:49.759341373Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:33:49.762959061Z","region_metadata_checked_at":"2026-09-23T15:33:49.762959061Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T13:22:24Z","started_at":"2026-09-23T13:22:24Z","completed_at":"2026-09-23T15:23:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:33:49.938102164Z","timing_metadata_checked_at":"2026-09-23T15:33:49.938102164Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:33:49.941253354Z","region_metadata_checked_at":"2026-09-23T15:33:49.941253354Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0a3f8e613d4d3ef59dc621da97f419de2b26a9953865953aded0a19e9cf54e06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","failed_at":"e2e","signature_id":"8eb0ca7714bb43bcafee5fa2084b2284a94351ac2160b6d55e893d478c2e48d8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094e860\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-nkx9tc\" in resourcegroup=\"rg-np-skip-minor-nkx9tc-svtsnwjlpglh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"name\": \"2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:30:28.639210211Z\",\n      \"endTime\": \"2026-09-23T14:49:28.972470638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094e860\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-nkx9tc\" in resourcegroup=\"rg-np-skip-minor-nkx9tc-svtsnwjlpglh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"name\": \"2b0016fd-fafe-4b72-8a66-7b759b311c62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:30:28.639210211Z\",\n      \"endTime\": \"2026-09-23T14:49:28.972470638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7378e7edfe7142a029cf5bd8d457c6e7e240863e3b7839d64527b18ccd60c2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102750533910532096","occurred_at":"2026-09-23T13:22:24Z","failed_at":"e2e","signature_id":"bccb01e525eebedfd6d8ea72bf2f185117e64e499bd98913c911974096d95997","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n      \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:433]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010240a0\u003e: \n    failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-w2f9x55rndpg\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"name\": \"269f9f7f-d982-42df-9b95-a4f0fcdc404f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T14:21:22.155873772Z\",\n      \"endTime\": \"2026-09-23T14:22:05.314078185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"4550f6295f01e13c8a713541787499b19d694a06","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T12:59:37Z","started_at":"2026-09-23T12:59:37Z","completed_at":"2026-09-23T14:49:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T15:02:24.20785241Z","timing_metadata_checked_at":"2026-09-23T15:02:24.20785241Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T15:02:24.2110087Z","region_metadata_checked_at":"2026-09-23T15:02:24.2110087Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ffe3d7b13f537077ac921f55036dfa0b03ee476b99c25331413cfdb82548e3b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102744695867183104","occurred_at":"2026-09-23T12:59:37Z","failed_at":"alert","signature_id":"f10cdf2980f54cb7434c563cea5c00835d86c81de376364c86add2439c33d20f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T13:47:46Z\n  Ended:       2026-09-23T14:19:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7183104-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7183104-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"open","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T12:35:31Z","started_at":"2026-09-23T12:35:31Z","completed_at":"2026-09-23T14:43:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:52:08.968389428Z","timing_metadata_checked_at":"2026-09-23T14:52:08.968389428Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:52:08.971641217Z","region_metadata_checked_at":"2026-09-23T14:52:08.971641217Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2ef9840825001e2a8318c3fc1eec97ce75bc6c2b6d3dcbb27256cb3c7d371262","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"alert","signature_id":"e8f9ea179e852213d3a81d66ef232d1e14041b2de7883a9afe6b09a4073f601b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T14:09:57Z\n  Ended:       2026-09-23T14:15:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6650496-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6650496-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9752d5ac5dd7ded0de4eef211c7c8c920a8115c6c7a8088e373f4aa67e655663","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"e2e","signature_id":"0f9515a2db56a29e5a383d5fa728e947a2b0cee316b599fe636e25ce8c008556","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n      \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010253a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fpvkvnrfcfj2\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"name\": \"1688403e-4bb7-4250-94fc-34a807ea0027\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:47:50.262026457Z\",\n      \"endTime\": \"2026-09-23T14:06:54.054886757Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-v9rkf4k56z2c\" to finish creating: GET https://rp"}},{"row_id":"b5e5c1987a3648f8cc4f9fd2d02338b5cd1515495ded580221f4180c7dcafed1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"alert","signature_id":"c5b74c40552ae1b36fab99026dc41a2a915f1d1a792b500821eca9610ee3d31e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T14:29:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6650496-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wqp86\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-wqp86 (namespace arobit) on cluster ci01-j6650496-svc has averaged 125% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"ed52a45a00107d337dccfc29cf9418d334900ca6659eeabc51ebc9dfcbedddf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102738633516650496","occurred_at":"2026-09-23T12:35:31Z","failed_at":"e2e","signature_id":"a9f1409cf050d25886bb76839830611031e806e434f180cb1affd1b0a192d798","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ff9c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dzgz54ntn7hc\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"name\": \"184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:43:16.227264536Z\",\n      \"endTime\": \"2026-09-23T14:02:20.066830362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ff9c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dzgz54ntn7hc\" to finish creating: GET https://rp.aro-hcp-dev-shard0-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"name\": \"184bf507-60e3-4d4c-91bb-42f27f255323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T13:43:16.227264536Z\",\n      \"endTime\": \"2026-09-23T14:02:20.066830362Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7102/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102726486111817728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7102,"pr_state":"closed","pr_sha":"d9646c1471c3fa0c1dafff1b68b88cec57beaa45","final_merged_sha":"d9646c1471c3fa0c1dafff1b68b88cec57beaa45","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T11:47:15Z","started_at":"2026-09-23T11:47:15Z","completed_at":"2026-09-23T13:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T14:10:07.687404345Z","timing_metadata_checked_at":"2026-09-23T14:10:07.687404345Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T14:10:07.690643334Z","region_metadata_checked_at":"2026-09-23T14:10:07.690643334Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7103,"pr_state":"closed","pr_sha":"174cb485dd2d6fb1ec807a48bb1bb2d984e4c005","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T11:38:49Z","started_at":"2026-09-23T11:38:49Z","completed_at":"2026-09-23T13:35:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:48:58.160482071Z","timing_metadata_checked_at":"2026-09-23T13:48:58.160482071Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:48:58.175036223Z","region_metadata_checked_at":"2026-09-23T13:48:58.175036223Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"106ef20d0f435c213e1519425e760f4c9723af37cc0b17014b0d9c3e3fbf51f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"e2e","signature_id":"312be1f9783180426404eca8136660afa7c8940a066aacb369c351024c0827a4","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001a366b0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.20:59854-\u003e172.30.0.10:53: read: connection refused\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"541fd6b77a464bfc038d951dadcd2d23614700a714c9d71ec79413000191e475","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"alert","signature_id":"b94154bec553ae4d7de90c235afb929dc5f8241123d4ff71acb4b324e3664bec","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T13:20:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6900864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-st2mg\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-st2mg (namespace arobit) on cluster ci01-j6900864-svc has averaged 124.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"e184f90cd9d4a1e5055908b5b301b5ce9bbac921e2b0c1df5091b64f346f1ee9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7103/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102724367396900864","occurred_at":"2026-09-23T11:38:49Z","failed_at":"alert","signature_id":"66ef3a438e94ca88c6470f62824d14213734daf6ab2142316c1d734ddeaa9c0e","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:28:03Z\n  Ended:       2026-09-23T13:01:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6900864-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6900864-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7101/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102722381150687232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7101,"pr_state":"closed","pr_sha":"52944f9dea1b221bfcdc4f83c1349fea7ff49598","final_merged_sha":"52944f9dea1b221bfcdc4f83c1349fea7ff49598","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T11:30:56Z","started_at":"2026-09-23T11:30:56Z","completed_at":"2026-09-23T13:26:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:38:14.35999517Z","timing_metadata_checked_at":"2026-09-23T13:38:14.35999517Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:38:14.363564058Z","region_metadata_checked_at":"2026-09-23T13:38:14.363564058Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102722297386242048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"57066b220bb112a1dcac715da3b728883b34274f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T11:30:36Z","started_at":"2026-09-23T11:30:36Z","completed_at":"2026-09-23T13:40:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:48:58.007435382Z","timing_metadata_checked_at":"2026-09-23T13:48:58.007435382Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:48:58.013328962Z","region_metadata_checked_at":"2026-09-23T13:48:58.013328962Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T11:19:17Z","started_at":"2026-09-23T11:19:17Z","completed_at":"2026-09-23T13:19:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:28:04.761434518Z","timing_metadata_checked_at":"2026-09-23T13:28:04.761434518Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:28:04.765205806Z","region_metadata_checked_at":"2026-09-23T13:28:04.765205806Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6b6271afc67c6e8ee86afc9cf4b0788a0ccda653050ea6468a1ec1e9d667aca7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102719452461142016","occurred_at":"2026-09-23T11:19:17Z","failed_at":"alert","signature_id":"04cf9b464f041c4aa5f1606316891b1c2238b1191be6f182872d69d9ca55ffea","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:40:58Z\n  Ended:       2026-09-23T12:46:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1142016-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1142016-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"8e51fd56182fb53110559c6686373da1a4435f34","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T11:07:04Z","started_at":"2026-09-23T11:07:04Z","completed_at":"2026-09-23T13:08:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T13:17:40.999261135Z","timing_metadata_checked_at":"2026-09-23T13:17:40.999261135Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T13:17:41.002785522Z","region_metadata_checked_at":"2026-09-23T13:17:41.002785522Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a24a9058e17914bf6e7a75fc61011868b75db07b3fdad22bc056e572da02bbc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102716371392532480","occurred_at":"2026-09-23T11:07:04Z","failed_at":"alert","signature_id":"ecc22c0637cf2c040e23438995345ce727a3526729f1b45629b38665f2e2a492","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T12:03:55Z\n  Ended:       2026-09-23T12:37:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2532480-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2532480-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102701497664933888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T10:07:57Z","started_at":"2026-09-23T10:07:57Z","completed_at":"2026-09-23T12:24:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:35:09.233554853Z","timing_metadata_checked_at":"2026-09-23T12:35:09.233554853Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:35:09.236909042Z","region_metadata_checked_at":"2026-09-23T12:35:09.236909042Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102701493466435584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7099,"pr_state":"closed","pr_sha":"2208f09500e712bff63abf4822601511464809c3","final_merged_sha":"2208f09500e712bff63abf4822601511464809c3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T10:07:56Z","started_at":"2026-09-23T10:07:56Z","completed_at":"2026-09-23T12:13:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:25:05.974862745Z","timing_metadata_checked_at":"2026-09-23T12:25:05.974862745Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:25:05.97737014Z","region_metadata_checked_at":"2026-09-23T12:25:05.97737014Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"ceedf6dfd596d82f101784fbf903aaa984937b18","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T10:05:03Z","started_at":"2026-09-23T10:05:03Z","completed_at":"2026-09-23T12:10:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T12:25:05.786905113Z","timing_metadata_checked_at":"2026-09-23T12:25:05.786905113Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T12:25:05.790614305Z","region_metadata_checked_at":"2026-09-23T12:25:05.790614305Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"68f489ba888e3d50dcca7a1d3b249eb56176cc6ca808f08bd1c7e5972bfae538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","failed_at":"e2e","signature_id":"80e228d9d0c46c7582a7eb683b187f1c62b13a5fdc9e873f52821f9b7629b50e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00190deb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00190deb0\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"a1867367778c664515dc8f00b2f77e472447bf8929f5a826301c9d56eb8acf3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102700766367059968","occurred_at":"2026-09-23T10:05:03Z","failed_at":"alert","signature_id":"695f2b58900eeb6ddc3f5c28514b819f16afd04fc4ec5e5122696e6f1efc6a9c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T11:26:05Z\n  Ended:       2026-09-23T11:37:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7059968-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7059968-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T09:32:18Z","started_at":"2026-09-23T09:32:18Z","completed_at":"2026-09-23T11:32:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:42:48.418910491Z","timing_metadata_checked_at":"2026-09-23T11:42:48.418910491Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:42:48.422422479Z","region_metadata_checked_at":"2026-09-23T11:42:48.422422479Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"672b3b5795cf93d757b5f7f36fe08af56dd457015a24b7919532a087a2f7659e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692523162996736","occurred_at":"2026-09-23T09:32:18Z","failed_at":"alert","signature_id":"5be4b79fd707f0f902f0c6190406a62281b77b6ad2601f0cc1b46e4e850c930c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:53:08Z\n  Ended:       2026-09-23T10:59:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2996736-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2996736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"863a5e13aa5f5eec1b54764ecc9ee558bc55f41c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:30:30Z","started_at":"2026-09-23T09:30:30Z","completed_at":"2026-09-23T11:27:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:42:48.608523644Z","timing_metadata_checked_at":"2026-09-23T11:42:48.608523644Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:42:48.611398134Z","region_metadata_checked_at":"2026-09-23T11:42:48.611398134Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"04ba43610abb0045a6381bb56babd96c5fa3abffacc42050543339ad68597cf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102692072241762304","occurred_at":"2026-09-23T09:30:30Z","failed_at":"alert","signature_id":"a430843224ed1479b519c2f15a1c5fc4265a858e6f819daf935153b2c4d45bf9","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:52:54Z\n  Ended:       2026-09-23T10:58:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1762304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1762304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7077,"pr_state":"closed","pr_sha":"54cd641541a0e7abd44ce2fadd761db8cd8f95ca","final_merged_sha":"53e6edac2bb7b25fcabf41daf657c529a89c7f7e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:20:17Z","started_at":"2026-09-23T09:20:17Z","completed_at":"2026-09-23T11:20:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:32:45.096592993Z","timing_metadata_checked_at":"2026-09-23T11:32:45.096592993Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:32:45.099124384Z","region_metadata_checked_at":"2026-09-23T11:32:45.099124384Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"383375b56891fc6d6be7c2490ff8f3cc89c9259a61ade5cc1cb16a8dc49240eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","failed_at":"alert","signature_id":"acc0593d70344c6ce5881c51325fa15f3ec8bcecedf6ebedda37a38bcf852bf6","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:25:13Z\n  Ended:       2026-09-23T10:52:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T10:25:14Z\n  Ended:       2026-09-23T10:54:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7833216-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7833216/providers/microsoft.monitor/accounts/hcps-j7833216\", namespace=\"ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-bj599\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t0lc7i6ae878vg6j5amjnqk5ucv64ji-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"885576e3a6fda561029ef56c863c71894b6435ef556c36dec718557fe5a1d1b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7077/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102689595517833216","occurred_at":"2026-09-23T09:20:17Z","failed_at":"e2e","signature_id":"7f41eb165d596d5dc7f2971efb22eb9df93680423e7b77c55b04839a62820eb7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001078700\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tsmxlnchqjdn\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-03.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"name\": \"5a99324f-ced3-4b58-b157-e5f91f1fdc43\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-23T10:04:12.923013072Z\",\n      \"endTime\": \"2026-09-23T10:33:15.61163171Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"f8180cd6044fd44b58d22fb50e2a7d9429087dfe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T09:17:52Z","started_at":"2026-09-23T09:17:52Z","completed_at":"2026-09-23T11:24:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:32:44.972534714Z","timing_metadata_checked_at":"2026-09-23T11:32:44.972534714Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:32:44.975882903Z","region_metadata_checked_at":"2026-09-23T11:32:44.975882903Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"21796c2413243b95cf2304b1af8be05d892462a25208ff8862c31e530ce07077","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102688892556677120","occurred_at":"2026-09-23T09:17:52Z","failed_at":"alert","signature_id":"168b46660d0ed6f4bf2e6d271e550940fff7e50d9ae0544d3d77fc3aaa3c674c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T10:47:31Z\n  Ended:       2026-09-23T10:53:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6677120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6677120-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102679333091938304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7070,"pr_state":"closed","pr_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","final_merged_sha":"10d1e6e67ad84132cdd528497cb78998e181d5f2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T08:39:52Z","started_at":"2026-09-23T08:39:52Z","completed_at":"2026-09-23T10:46:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T11:01:41.898232855Z","timing_metadata_checked_at":"2026-09-23T11:01:41.898232855Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T11:01:41.902008642Z","region_metadata_checked_at":"2026-09-23T11:01:41.902008642Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T08:02:37Z","started_at":"2026-09-23T08:02:37Z","completed_at":"2026-09-23T10:11:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T10:30:38.200722845Z","timing_metadata_checked_at":"2026-09-23T10:30:38.200722845Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T10:30:38.203847735Z","region_metadata_checked_at":"2026-09-23T10:30:38.203847735Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"80918aeb625d32fdb0e8534a588ebd2d48f0c1da28fe81b6d18ca6c9ea3b1d01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669953994854400","occurred_at":"2026-09-23T08:02:37Z","failed_at":"alert","signature_id":"ef8ed7f81b6e444cb9f47b23d2f051a1e779787724507e05e79e426102d41cd3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T09:00:54Z\n  Ended:       2026-09-23T09:34:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4854400-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4854400-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T08:02:34Z","started_at":"2026-09-23T08:02:34Z","completed_at":"2026-09-23T10:02:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T10:20:29.53596881Z","timing_metadata_checked_at":"2026-09-23T10:20:29.53596881Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T10:20:29.539446698Z","region_metadata_checked_at":"2026-09-23T10:20:29.539446698Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0293d84de252cc10fa4aba8b36b0124251c3aaff37c8e07cdc46188972b5f89d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102669943756558336","occurred_at":"2026-09-23T08:02:34Z","failed_at":"alert","signature_id":"6a553ba910a4f3a893c5726596a17745a39160fc0a64f583a6cc5c05a5a4ea9e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T09:48:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6558336-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j66dp\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-j66dp (namespace arobit) on cluster ci01-j6558336-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102659427256504320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7072,"pr_state":"closed","pr_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","final_merged_sha":"a1bb52523ea8e8e012393f45ec7f563124c2deed","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T07:20:47Z","started_at":"2026-09-23T07:20:47Z","completed_at":"2026-09-23T07:30:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:36:51.480167516Z","timing_metadata_checked_at":"2026-09-23T07:36:51.480167516Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T07:36:51.482770407Z","region_metadata_checked_at":"2026-09-23T08:40:09.114907986Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"45cadb2e34fb786fe9639fb0838e0d13c322eaa13adc1aa7f611ad31e0a8a4f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7072/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102659427256504320","occurred_at":"2026-09-23T07:20:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/4878/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102655845220225024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4878,"pr_state":"closed","pr_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","final_merged_sha":"244d0c16bcabe8284755cf089f7bb166cd38a435","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T07:06:33Z","started_at":"2026-09-23T07:06:33Z","completed_at":"2026-09-23T09:03:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T09:17:56.890736447Z","timing_metadata_checked_at":"2026-09-23T09:17:56.890736447Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T09:17:56.893966036Z","region_metadata_checked_at":"2026-09-23T09:17:56.893966036Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"1769127263098290ba69f6f4e9bc0696f1c66e33","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:50:00Z","started_at":"2026-09-23T06:50:00Z","completed_at":"2026-09-23T08:55:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:57.542614074Z","timing_metadata_checked_at":"2026-09-23T08:56:57.542614074Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:57.54614706Z","region_metadata_checked_at":"2026-09-23T08:56:57.54614706Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c1e6ae9e98fc46a47884a91a6f65821208f2208f21b1b8fa8fd0373da726bcf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651678955147264","occurred_at":"2026-09-23T06:50:00Z","failed_at":"alert","signature_id":"01cbd1d336e53f82a367140655d108e641e536db1311bf518e94e91e812f8a75","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:33:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j5147264-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j5147264-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"open","pr_sha":"b6ef3476b2950b7c03a8b9bbd77712889cebd2df","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:49:08Z","started_at":"2026-09-23T06:49:08Z","completed_at":"2026-09-23T09:00:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T09:17:56.908758485Z","timing_metadata_checked_at":"2026-09-23T09:17:56.908758485Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T09:17:56.911433976Z","region_metadata_checked_at":"2026-09-23T09:17:56.911433976Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"313a8e1451063175a424d0d816daef49504688162aa45df25cdcc656d0471bcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"alert","signature_id":"61fa061764ad73e8317dc73f0ca153b35f59fa073234e6998c92f96b627530c7","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:47:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3684224-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z8dk5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-z8dk5 (namespace arobit) on cluster ci01-j3684224-svc has averaged 125.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"52d9bb0ef79dcf0073fa404bc89036b655c5d0aaf88ae71236091f8ddf4b2596","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"e2e","signature_id":"a02ab509af0dce2add587b938aa804dece2558ca7621ffc208e009f99dae4089","test_name":"Customer should upgrade control plane minor version when dataplane is degraded from 4.20 minor to 4.21 minor with cordoned nodes","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_dataplane.go:304]: guest nodes were no longer unschedulable after upgrade on cluster \"cp-ystream-dp-cordon-4-21-8ndsmf\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013005c0\u003e: \n    expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","failure_pattern":"expected all guest nodes to be unschedulable, schedulable=\u003cnode\u003e, \u003cnode\u003e; nodes: \u003cnode\u003e ready=true unschedulable=true created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=true unschedulable=false created=\u003ctimestamp\u003e nodePool=\u003cnodepool\u003e; \u003cnode\u003e ready=false","search_query":"expected all guest nodes to be unschedulable, schedulable=e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv, e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r; nodes: e6r1q1p6z7w9f4r-np-dataplane-fqlcw-4wrz9 ready=true unschedulable=true created=2026-09-23T07:52:57Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-fvjmv ready=true unschedulable=false created=2026-09-23T08:05:13Z nodePool=e6r1q1p6z7w9f4r-np-dataplane; e6r1q1p6z7w9f4r-np-dataplane-wct59-nqv5r ready=false unschedulable=false created=2026-09-23T08:11:10Z nodePool=e6r1q1p6z7w9f4r-np-dataplane"}},{"row_id":"e1424293e6ef937d8f58a7b1246d19e995f47c884731c8311815c0385ee46e3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102651461333684224","occurred_at":"2026-09-23T06:49:08Z","failed_at":"alert","signature_id":"24389b82f6419bf8544e5612ab91556f9033a43c11c58c8a75f9eddd0d3cb5d2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:11:35Z\n  Ended:       2026-09-23T08:22:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3684224-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3684224-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["61e8157ef3845c610ccadbb9e49365372ea07610a9b75316502d11b44c89c896","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7099,"pr_state":"closed","pr_sha":"2208f09500e712bff63abf4822601511464809c3","final_merged_sha":"2208f09500e712bff63abf4822601511464809c3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T06:46:50Z","started_at":"2026-09-23T06:46:50Z","completed_at":"2026-09-23T08:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:58.215393187Z","timing_metadata_checked_at":"2026-09-23T08:56:58.215393187Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:58.218456976Z","region_metadata_checked_at":"2026-09-23T08:56:58.218456976Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"294ca3dfd83b32cf8128240e53199485e5b32182c9d77acf19b518b1387f1c93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","failed_at":"alert","signature_id":"7b5a0fe8e7d62a0661702614b741d3d10ab9bf69cb3d6aad1f5d3a0807f9d130","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:39:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j3407488-svc\", component=\"slo\", long_window=\"6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n  Description: More than 6% of Frontend requests on cluster ci01-j3407488-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}},{"row_id":"33cf1cfbd556ae52ff52d99d61dcedc1b01d5be2c1e8939a0b304abfc63d5c57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7099/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650881433407488","occurred_at":"2026-09-23T06:46:50Z","failed_at":"alert","signature_id":"7c059f6930e4e70e01cf60cdc5a209a41e1d36e3c5cdc0a3889876307b50e19b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:08:37Z\n  Ended:       2026-09-23T08:22:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3407488-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j3407488-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"57066b220bb112a1dcac715da3b728883b34274f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T06:44:17Z","started_at":"2026-09-23T06:44:17Z","completed_at":"2026-09-23T08:53:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:56:57.863225578Z","timing_metadata_checked_at":"2026-09-23T08:56:57.863225578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:56:57.865784969Z","region_metadata_checked_at":"2026-09-23T08:56:57.865784969Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"6e7f9e6dca9ada1bde642f56305a573d151deac2df7e1fdedc4cee00a17e8fe1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","failed_at":"alert","signature_id":"9b7c4ba8bc11e40e7d6443d7bd87ac580a2605e35dba3d0581d0e43dcef62d3d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T08:11:20Z\n  Ended:       2026-09-23T08:18:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5172864-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5172864-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"ce8f0e173cb61e12360c71d260495a0a6f1c2a8281fe73d298d184e3ffb4eb87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102650243035172864","occurred_at":"2026-09-23T06:44:17Z","failed_at":"alert","signature_id":"398824eec9025702d224d5a9ba36e64c2703dfdc5e00ba0eb6afbc886a9d202d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T08:39:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5172864-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j4msz\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-j4msz (namespace arobit) on cluster ci01-j5172864-svc has averaged 124.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7051/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102639009233637376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7051,"pr_state":"open","pr_sha":"e7b5fbaebd5615c4c4ed6f61605b5b8811a4947c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-23T05:59:38Z","started_at":"2026-09-23T05:59:38Z","completed_at":"2026-09-23T08:10:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T08:18:41.894413233Z","timing_metadata_checked_at":"2026-09-23T08:18:41.894413233Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T08:18:41.900036514Z","region_metadata_checked_at":"2026-09-23T08:18:41.900036514Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T05:02:00Z","started_at":"2026-09-23T05:02:00Z","completed_at":"2026-09-23T07:13:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:26:35.782975741Z","timing_metadata_checked_at":"2026-09-23T07:26:35.782975741Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T07:26:35.78648243Z","region_metadata_checked_at":"2026-09-23T07:26:35.78648243Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"86057d21266144a0386954db6763cf7e49579e2c47a6c299ae788a922b6f3b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624700403290112","occurred_at":"2026-09-23T05:02:00Z","failed_at":"alert","signature_id":"fec44601492cd080815bfd1f2fda9d583e0a942b12d71ab0789e80aa75b0494f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T06:11:33Z\n  Ended:       2026-09-23T06:20:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3290112-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y\", pod=\"community-operators-catalog-5f9d56fbbf-bvthw\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t0homk1jhbv7gpr72fmt0jahgksipbf-m8t7b7k5f6z0o9y/community-operators-catalog-5f9d56fbbf-bvthw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T05:01:42Z","started_at":"2026-09-23T05:01:42Z","completed_at":"2026-09-23T07:17:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T07:26:35.907242631Z","timing_metadata_checked_at":"2026-09-23T07:26:35.907242631Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T07:26:35.910308421Z","region_metadata_checked_at":"2026-09-23T07:26:35.910308421Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"38f088f8d1d9f1694bec5e46d0e1b3eae1afd9b68c8e98a1bc85be6786d5f064","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102624604781547520","occurred_at":"2026-09-23T05:01:42Z","failed_at":"alert","signature_id":"e61751d58352ab491f0598c208a923b5d0b7b5b03f412d966b5363851718b33b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T06:43:51Z\n  Ended:       2026-09-23T06:49:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1547520-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1547520-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102620674894860288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T04:46:48Z","started_at":"2026-09-23T04:46:48Z","completed_at":"2026-09-23T04:55:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T05:11:21.468834083Z","timing_metadata_checked_at":"2026-09-23T05:11:21.468834083Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-23T05:11:21.472427671Z","region_metadata_checked_at":"2026-09-23T06:12:21.316214356Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"db9ef5b98ba20063f2855cc462d10730ad69b3e2601efcf25eb480fd4725b169","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102620674894860288","occurred_at":"2026-09-23T04:46:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7098/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579983321927680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7098,"pr_state":"closed","pr_sha":"f863469794c6e0ad6c0a4d8c8fd4f2a8051647db","final_merged_sha":"f863469794c6e0ad6c0a4d8c8fd4f2a8051647db","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T02:05:06Z","started_at":"2026-09-23T02:05:06Z","completed_at":"2026-09-23T04:11:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:17:58.158395822Z","timing_metadata_checked_at":"2026-09-23T04:17:58.158395822Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:17:58.161533612Z","region_metadata_checked_at":"2026-09-23T04:17:58.161533612Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T02:01:18Z","started_at":"2026-09-23T02:01:18Z","completed_at":"2026-09-23T04:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:07:14.580204121Z","timing_metadata_checked_at":"2026-09-23T04:07:14.580204121Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:07:14.582789412Z","region_metadata_checked_at":"2026-09-23T04:07:14.582789412Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"0eb1ca3df25acb4111a6e19897c009e53a9ba98abc3b19f34fc90de0cfbe6cf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"b52a16123591cbdade9825eddb9e37de34eaab046973c4b8e648b0ae88335846","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T03:16:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 250ms over the past 30 minutes (cluster ci01-j3482624-svc).\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-23T03:29:12Z\n  Ended:       2026-09-23T03:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3482624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9dcd032efeed8c80f0a430d4e4579b817e20e336e340f47fbff13e9c5a927e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T03:10:35Z\n  Ended:       2026-09-23T03:20:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3482624-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"a8d424a7f632829120e0f9b5b5977b2c2480a331622f269b3e60a103470b27bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579128883482624","occurred_at":"2026-09-23T02:01:18Z","failed_at":"alert","signature_id":"b20cadb846b55234efc43634cdf321fda9be13c6186765f74dd9bcf81c873e27","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-23T03:37:03Z\n  Ended:       2026-09-23T03:47:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3482624-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j3482624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7089/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102579123015651328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7089,"pr_state":"closed","pr_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","final_merged_sha":"0a81896e19a7bd7d095267a0eb3a19c7d7fd94e9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T02:01:11Z","started_at":"2026-09-23T02:01:11Z","completed_at":"2026-09-23T04:08:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T04:17:58.031253737Z","timing_metadata_checked_at":"2026-09-23T04:17:58.031253737Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T04:17:58.034839626Z","region_metadata_checked_at":"2026-09-23T04:17:58.034839626Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-23T01:27:26Z","started_at":"2026-09-23T01:27:26Z","completed_at":"2026-09-23T03:17:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T03:25:11.165946146Z","timing_metadata_checked_at":"2026-09-23T03:25:11.165946146Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T03:25:11.169534334Z","region_metadata_checked_at":"2026-09-23T03:25:11.169534334Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"084d748eb8dde746c988f9a89fea25877605d507aad827cb31079b3f1adf251e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102570502110318592","occurred_at":"2026-09-23T01:27:26Z","failed_at":"e2e","signature_id":"75beab8f572182a387939937ea4f447a02998cc8ddb2ad7397436d6df460ffc0","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f44210\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-7pwjpcnwnctq\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-7pwjpcnwnctq, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-23T01:02:20Z","started_at":"2026-09-23T01:02:20Z","completed_at":"2026-09-23T02:21:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:34:36.179192523Z","timing_metadata_checked_at":"2026-09-23T02:34:36.179192523Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:34:36.182802611Z","region_metadata_checked_at":"2026-09-23T02:34:36.182802611Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"97a00ad8ca6ddca5e4e12c96f8c3a591af8cbd5c5474f9ae07e31cffe0e93ee7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"cd2acdca3a5c4271f034e124fe4ffb19ca10970d45a2e64d9db71f66d0c505d1","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-ch8gg\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-ch8gg has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-9ptjq\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-9ptjq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-kw5rf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-kw5rf has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T01:52:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-cxgg8\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-cxgg8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"c3227c73c9e918743470589e845ca9073ba8bb9cb27f129d4b5eebc2a1c0db91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"provision","signature_id":"f020f9342f43cace837cc83b8952affa433a8c7410f5746e59533d769ed278b9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-23T01:45:47.799Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:45:48.393Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:48.667Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:45:49.209Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:45:49.334Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:55:54.330Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:55:54.331Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T01:56:54.333Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T01:56:55.421Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:55.777Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.324Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:56.630Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T01:56:57.690Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:56:57.691Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.299Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:56:58.862Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:56:59.423Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T01:57:00.012Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.364Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:00.910Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T01:57:01.301Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T01:57:01.546Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-23T02:07:04.175Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:07:04.175Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-23T02:08:04.175Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0769280-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j0769280-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5eca8f713c64383bc545dd55f4af4020fd5db0c392d891cdc441765a2f7e52c3 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:1190e8af-ea6a-4410-bc47-5facec337dc4 region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:1fa09488bf407f4b1bfd44f02ff60af191218075be0a2050d737165acfcbda23 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#1190e8af-ea6a-4410-bc47-5facec337dc4 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-23T02:08:05.008Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:05.466Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.204Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:06.382Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:07.458Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:07.843Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:08.593Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.447Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-23T02:08:09.853Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:10.521Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-23T02:08:11.889Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-23T02:08:12.134Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-23T02:15:47.989Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-23T02:15:47.990Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"eb5a7eb26294cc47e37765b4b6c6d557f498ff7a6d3497d50563ce0fd68440f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"80bbd3a02e54b8fc7971960324a0123e49b270a5d705be23d017d8e1a4b2f32e","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0769280-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0769280/providers/microsoft.monitor/accounts/services-j0769280\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-wqtm5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"ec99c812f36f32216c370a3ebe0c6c035e70c262b41bc386f59c11dce901f8a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564185580769280","occurred_at":"2026-09-23T01:02:20Z","failed_at":"alert","signature_id":"0bff05a4b68cd34010dac9320f26f4a4adb189546a09d7a89930f18e9ffb6349","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-23T02:02:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0769280-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j0769280-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7064/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102564169285898240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7064,"pr_state":"closed","pr_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","final_merged_sha":"f79a2b49b041574b466c35db9b025bacfc78ce0d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T01:02:16Z","started_at":"2026-09-23T01:02:16Z","completed_at":"2026-09-23T03:01:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T03:14:58.015236856Z","timing_metadata_checked_at":"2026-09-23T03:14:58.015236856Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T03:14:58.018693443Z","region_metadata_checked_at":"2026-09-23T03:14:58.018693443Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7085/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102561003798532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7085,"pr_state":"closed","pr_sha":"b8e74897ad36b238125a1aefbaf68e7eae50c7d9","final_merged_sha":"b8e74897ad36b238125a1aefbaf68e7eae50c7d9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T00:49:41Z","started_at":"2026-09-23T00:49:41Z","completed_at":"2026-09-23T02:39:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:44:49.484343071Z","timing_metadata_checked_at":"2026-09-23T02:44:49.484343071Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:44:49.488061359Z","region_metadata_checked_at":"2026-09-23T02:44:49.488061359Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102549176721084416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5846,"pr_state":"closed","pr_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","final_merged_sha":"e8df860b9cf41bfcff7941a3a70620c3c2689800","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-23T00:02:31Z","started_at":"2026-09-23T00:02:31Z","completed_at":"2026-09-23T02:06:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-23T02:13:15.093950874Z","timing_metadata_checked_at":"2026-09-23T02:13:15.093950874Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-23T02:13:15.097662262Z","region_metadata_checked_at":"2026-09-23T02:13:15.097662262Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}